[sample-tester] Collect app size and build duration statistics. (#7602)
* [sample-tester] Collect app size and build duration statistics. * Always publish performance data. * [mmp] Add support for MMP_ENV_OPTIONS to mirror mtouch's MTOUCH_ENV_OPTIONS. * [sample-tester] Make mmp/mtouch show timing information, and get the diagnostic msbuild log. * Collect task and target info. * Add more perf data and a baseline test. * Redo the base test a bit. * More tweaks. * More tweaks. * Huh? * Debug stuff. * diagnostics. * Better diagnostics. * cleanup * tmp * Use a separate stage to push data. * Next attempt. * Unique artifact names. * Make the right name unique. * More progress * Improvements * Don't need new maccore. * cleanup * Remove debug spew. * Realign stuff. * duh * More path fixes. * zippity zip. * Show publishing errors. * dependencies * Not fake results anymore. * Tweaks * Merge xml files * [sampletester] Bump nuget timeout to 5 minutes. The TodoREST.iOS test seems to use a lot of packages, and it can apparently take a while to restore them, so give a few more minutes. * Improvements. * Fix glob. * [sampletester] Fix fetching of sample repository and hash. * [sampletester] Centralize verbosity for mtouch and mmp.
This commit is contained in:
Rolf Bjarne Kvinge
Sebastien Pouliot
Родитель
fa00d7a669
Коммит
1d0dd15bd4
|
|
@ -0,0 +1,28 @@
|
|||
using Foundation;
|
||||
using UIKit;
|
||||
|
||||
namespace BaselineTest {
|
||||
[Register ("AppDelegate")]
|
||||
public class AppDelegate : UIResponder, IUIApplicationDelegate {
|
||||
UIWindow window;
|
||||
UIViewController vc;
|
||||
|
||||
[Export ("application:didFinishLaunchingWithOptions:")]
|
||||
public bool FinishedLaunching (UIApplication application, NSDictionary launchOptions)
|
||||
{
|
||||
window = new UIWindow (UIScreen.MainScreen.Bounds);
|
||||
vc = new UIViewController ();
|
||||
vc.View.BackgroundColor = UIColor.Green;
|
||||
window.RootViewController = vc;
|
||||
window.MakeKeyAndVisible ();
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static void Main (string [] args)
|
||||
{
|
||||
UIApplication.Main (args, null, typeof (AppDelegate));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<PropertyGroup>
|
||||
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
||||
<Platform Condition=" '$(Platform)' == '' ">iPhoneSimulator</Platform>
|
||||
<ProjectGuid>{E7E76A89-6DB7-4CF6-953C-6D9ADC656C6F}</ProjectGuid>
|
||||
<ProjectTypeGuids>{FEACFBD2-3405-455C-9665-78FE426C6842};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}</ProjectTypeGuids>
|
||||
<TemplateGuid>{edc1b0fa-90cd-4038-8fad-98fe74adb368}</TemplateGuid>
|
||||
<OutputType>Exe</OutputType>
|
||||
<RootNamespace>BaselineTest</RootNamespace>
|
||||
<AssemblyName>BaselineTest</AssemblyName>
|
||||
<IPhoneResourcePrefix>Resources</IPhoneResourcePrefix>
|
||||
<MtouchEnableSGenConc>true</MtouchEnableSGenConc>
|
||||
<MtouchHttpClientHandler>NSUrlSessionHandler</MtouchHttpClientHandler>
|
||||
<RestoreProjectStyle>PackageReference</RestoreProjectStyle>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|iPhoneSimulator' ">
|
||||
<DebugSymbols>true</DebugSymbols>
|
||||
<DebugType>full</DebugType>
|
||||
<Optimize>false</Optimize>
|
||||
<OutputPath>bin\iPhoneSimulator\Debug</OutputPath>
|
||||
<DefineConstants>DEBUG</DefineConstants>
|
||||
<ErrorReport>prompt</ErrorReport>
|
||||
<WarningLevel>4</WarningLevel>
|
||||
<MtouchArch>x86_64</MtouchArch>
|
||||
<MtouchLink>None</MtouchLink>
|
||||
<MtouchDebug>true</MtouchDebug>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|iPhoneSimulator' ">
|
||||
<DebugType>none</DebugType>
|
||||
<Optimize>true</Optimize>
|
||||
<OutputPath>bin\iPhoneSimulator\Release</OutputPath>
|
||||
<ErrorReport>prompt</ErrorReport>
|
||||
<WarningLevel>4</WarningLevel>
|
||||
<MtouchLink>None</MtouchLink>
|
||||
<MtouchArch>x86_64</MtouchArch>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|iPhone' ">
|
||||
<DebugSymbols>true</DebugSymbols>
|
||||
<DebugType>full</DebugType>
|
||||
<Optimize>false</Optimize>
|
||||
<OutputPath>bin\iPhone\Debug</OutputPath>
|
||||
<DefineConstants>DEBUG</DefineConstants>
|
||||
<ErrorReport>prompt</ErrorReport>
|
||||
<WarningLevel>4</WarningLevel>
|
||||
<MtouchArch>ARM64</MtouchArch>
|
||||
<CodesignKey>iPhone Developer</CodesignKey>
|
||||
<MtouchDebug>true</MtouchDebug>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|iPhone' ">
|
||||
<DebugType>none</DebugType>
|
||||
<Optimize>true</Optimize>
|
||||
<OutputPath>bin\iPhone\Release</OutputPath>
|
||||
<ErrorReport>prompt</ErrorReport>
|
||||
<WarningLevel>4</WarningLevel>
|
||||
<MtouchArch>ARM64</MtouchArch>
|
||||
<CodesignKey>iPhone Developer</CodesignKey>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<Reference Include="Xamarin.iOS" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<InterfaceDefinition Include="LaunchScreen.storyboard" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<None Include="Info.plist" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Compile Include="AppDelegate.cs" />
|
||||
</ItemGroup>
|
||||
<Import Project="$(MSBuildExtensionsPath)\Xamarin\iOS\Xamarin.iOS.CSharp.targets" />
|
||||
</Project>
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
|
||||
Microsoft Visual Studio Solution File, Format Version 12.00
|
||||
# Visual Studio 15
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BaselineTest", "BaselineTest.csproj", "{E7E76A89-6DB7-4CF6-953C-6D9ADC656C6F}"
|
||||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||
Debug|iPhoneSimulator = Debug|iPhoneSimulator
|
||||
Release|iPhoneSimulator = Release|iPhoneSimulator
|
||||
Debug|iPhone = Debug|iPhone
|
||||
Release|iPhone = Release|iPhone
|
||||
EndGlobalSection
|
||||
GlobalSection(ProjectConfigurationPlatforms) = postSolution
|
||||
{E7E76A89-6DB7-4CF6-953C-6D9ADC656C6F}.Debug|iPhoneSimulator.ActiveCfg = Debug|iPhoneSimulator
|
||||
{E7E76A89-6DB7-4CF6-953C-6D9ADC656C6F}.Debug|iPhoneSimulator.Build.0 = Debug|iPhoneSimulator
|
||||
{E7E76A89-6DB7-4CF6-953C-6D9ADC656C6F}.Release|iPhoneSimulator.ActiveCfg = Release|iPhoneSimulator
|
||||
{E7E76A89-6DB7-4CF6-953C-6D9ADC656C6F}.Release|iPhoneSimulator.Build.0 = Release|iPhoneSimulator
|
||||
{E7E76A89-6DB7-4CF6-953C-6D9ADC656C6F}.Debug|iPhone.ActiveCfg = Debug|iPhone
|
||||
{E7E76A89-6DB7-4CF6-953C-6D9ADC656C6F}.Debug|iPhone.Build.0 = Debug|iPhone
|
||||
{E7E76A89-6DB7-4CF6-953C-6D9ADC656C6F}.Release|iPhone.ActiveCfg = Release|iPhone
|
||||
{E7E76A89-6DB7-4CF6-953C-6D9ADC656C6F}.Release|iPhone.Build.0 = Release|iPhone
|
||||
EndGlobalSection
|
||||
EndGlobal
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>CFBundleDisplayName</key>
|
||||
<string>BaselineTest</string>
|
||||
<key>CFBundleIdentifier</key>
|
||||
<string>com.xamarin.BaselineTest</string>
|
||||
<key>CFBundleShortVersionString</key>
|
||||
<string>1.0</string>
|
||||
<key>CFBundleVersion</key>
|
||||
<string>1.0</string>
|
||||
<key>LSRequiresIPhoneOS</key>
|
||||
<true/>
|
||||
<key>MinimumOSVersion</key>
|
||||
<string>8.0</string>
|
||||
<key>UIDeviceFamily</key>
|
||||
<array>
|
||||
<integer>1</integer>
|
||||
<integer>2</integer>
|
||||
</array>
|
||||
<key>UILaunchStoryboardName</key>
|
||||
<string>LaunchScreen</string>
|
||||
<key>UIMainStoryboardFile~ipad</key>
|
||||
<string>Main</string>
|
||||
<key>UIRequiredDeviceCapabilities</key>
|
||||
<array>
|
||||
<string>armv7</string>
|
||||
</array>
|
||||
<key>UISupportedInterfaceOrientations</key>
|
||||
<array>
|
||||
<string>UIInterfaceOrientationPortrait</string>
|
||||
<string>UIInterfaceOrientationLandscapeLeft</string>
|
||||
<string>UIInterfaceOrientationLandscapeRight</string>
|
||||
</array>
|
||||
<key>UISupportedInterfaceOrientations~ipad</key>
|
||||
<array>
|
||||
<string>UIInterfaceOrientationPortrait</string>
|
||||
<string>UIInterfaceOrientationPortraitUpsideDown</string>
|
||||
<string>UIInterfaceOrientationLandscapeLeft</string>
|
||||
<string>UIInterfaceOrientationLandscapeRight</string>
|
||||
</array>
|
||||
<key>XSAppIconAssets</key>
|
||||
<string>Assets.xcassets/AppIcon.appiconset</string>
|
||||
<key>CFBundleName</key>
|
||||
<string>BaselineTest</string>
|
||||
</dict>
|
||||
</plist>
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="9532" systemVersion="15D21" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" launchScreen="YES" useTraitCollections="YES" initialViewController="01J-lp-oVM">
|
||||
<dependencies>
|
||||
<deployment identifier="iOS" />
|
||||
<plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="9530" />
|
||||
</dependencies>
|
||||
<scenes>
|
||||
<!--View Controller-->
|
||||
<scene sceneID="EHf-IW-A2E">
|
||||
<objects>
|
||||
<viewController id="01J-lp-oVM" sceneMemberID="viewController">
|
||||
<layoutGuides>
|
||||
<viewControllerLayoutGuide type="top" id="Llm-lL-Icb" />
|
||||
<viewControllerLayoutGuide type="bottom" id="xb3-aO-Qok" />
|
||||
</layoutGuides>
|
||||
<view key="view" contentMode="scaleToFill" id="Ze5-6b-2t3">
|
||||
<rect key="frame" x="0.0" y="0.0" width="600" height="600" />
|
||||
<autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES" />
|
||||
<color key="backgroundColor" white="1" alpha="1" colorSpace="custom" customColorSpace="calibratedWhite" />
|
||||
</view>
|
||||
</viewController>
|
||||
<placeholder placeholderIdentifier="IBFirstResponder" id="iYj-Kq-Ea1" userLabel="First Responder" sceneMemberID="firstResponder" />
|
||||
</objects>
|
||||
<point key="canvasLocation" x="53" y="375" />
|
||||
</scene>
|
||||
</scenes>
|
||||
</document>
|
||||
|
|
@ -52,5 +52,57 @@ namespace Xamarin.Tests {
|
|||
}
|
||||
");
|
||||
}
|
||||
|
||||
static string tested_hash;
|
||||
public static string TestedHash {
|
||||
get {
|
||||
lock (lock_obj) {
|
||||
if (tested_hash != null)
|
||||
return tested_hash;
|
||||
|
||||
tested_hash = GetCurrentHash (Environment.CurrentDirectory);
|
||||
return tested_hash;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static string GetCurrentHash (string directory)
|
||||
{
|
||||
return ProcessHelper.RunProcess ("git", "log -1 --pretty=%H", directory).Trim ();
|
||||
}
|
||||
|
||||
public static string GetCurrentRemoteUrl (string directory)
|
||||
{
|
||||
return ProcessHelper.RunProcess ("git", "remote get-url origin", directory).Trim ();
|
||||
}
|
||||
|
||||
static string mono_version;
|
||||
public static string MonoVersion {
|
||||
get {
|
||||
lock (lock_obj) {
|
||||
if (mono_version != null)
|
||||
return mono_version;
|
||||
|
||||
// We only care about the first line
|
||||
mono_version = ProcessHelper.RunProcess ("mono", "--version").Split (new char [] { '\n' }, StringSplitOptions.RemoveEmptyEntries) [0].Trim ();
|
||||
}
|
||||
|
||||
return mono_version;
|
||||
}
|
||||
}
|
||||
|
||||
static string sw_version;
|
||||
public static string OSVersion {
|
||||
get {
|
||||
lock (lock_obj) {
|
||||
if (sw_version != null)
|
||||
return sw_version;
|
||||
|
||||
sw_version = ProcessHelper.RunProcess ("sw_vers").Replace ('\n', ';').Replace ((char) 9, ' ');
|
||||
}
|
||||
|
||||
return sw_version;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ using System.Linq;
|
|||
using System.Threading;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Xml;
|
||||
|
||||
using NUnit.Framework;
|
||||
|
||||
|
|
@ -28,6 +29,11 @@ public static class ProcessHelper
|
|||
}
|
||||
|
||||
public static void AssertRunProcess (string filename, string[] arguments, TimeSpan timeout, string workingDirectory, Dictionary<string, string> environment_variables, string message)
|
||||
{
|
||||
AssertRunProcess (filename, arguments, timeout, workingDirectory, environment_variables, message, out _);
|
||||
}
|
||||
|
||||
public static void AssertRunProcess (string filename, string[] arguments, TimeSpan timeout, string workingDirectory, Dictionary<string, string> environment_variables, string message, out string logfile)
|
||||
{
|
||||
var exitCode = 0;
|
||||
var output = new List<string> ();
|
||||
|
|
@ -49,7 +55,7 @@ public static class ProcessHelper
|
|||
output_callback ($"Exit code: {exitCode} Timed out: {timed_out} Total duration: {watch.Elapsed.ToString ()}");
|
||||
|
||||
// Write execution log to disk (and print the path)
|
||||
var logfile = Path.Combine (LogDirectory, $"{filename}-{Interlocked.Increment (ref counter)}.log");
|
||||
logfile = Path.Combine (LogDirectory, $"{filename}-{Interlocked.Increment (ref counter)}.log");
|
||||
File.WriteAllLines (logfile, output);
|
||||
TestContext.AddTestAttachment (logfile, $"Execution log for {filename}");
|
||||
Console.WriteLine ("Execution log for {0}: {1}", filename, logfile);
|
||||
|
|
@ -79,8 +85,8 @@ public static class ProcessHelper
|
|||
nuget_args.Add ("sln"); // replaced later
|
||||
nuget_args.Add ("-Verbosity");
|
||||
nuget_args.Add ("detailed");
|
||||
var slndir = Path.GetDirectoryName (solution);
|
||||
if (!solution.EndsWith (".sln", StringComparison.Ordinal)) {
|
||||
var slndir = Path.GetDirectoryName (solution);
|
||||
while ((solutions = Directory.GetFiles (slndir, "*.sln", SearchOption.TopDirectoryOnly)).Length == 0 && slndir.Length > 1)
|
||||
slndir = Path.GetDirectoryName (slndir);
|
||||
nuget_args.Add ("-SolutionDir");
|
||||
|
|
@ -89,7 +95,7 @@ public static class ProcessHelper
|
|||
|
||||
foreach (var sln in solutions) {
|
||||
nuget_args [1] = sln; // replacing here
|
||||
AssertRunProcess ("nuget", nuget_args.ToArray (), TimeSpan.FromMinutes (2), Configuration.SampleRootDirectory, environment_variables, "nuget restore");
|
||||
AssertRunProcess ("nuget", nuget_args.ToArray (), TimeSpan.FromMinutes (5), Configuration.SampleRootDirectory, environment_variables, "nuget restore");
|
||||
}
|
||||
|
||||
// msbuild
|
||||
|
|
@ -102,6 +108,201 @@ public static class ProcessHelper
|
|||
sb.Add (solution);
|
||||
if (!string.IsNullOrEmpty (target))
|
||||
sb.Add ($"/t:{target}");
|
||||
AssertRunProcess ("msbuild", sb.ToArray (), TimeSpan.FromMinutes (5), Configuration.SampleRootDirectory, environment_variables, "build");
|
||||
|
||||
sb.Add ($"/verbosity:diag");
|
||||
environment_variables ["MTOUCH_ENV_OPTIONS"] = "--time --time --time --time -vvvv";
|
||||
environment_variables ["MMP_ENV_OPTIONS"] = "--time --time --time --time -vvvv";
|
||||
|
||||
var watch = Stopwatch.StartNew ();
|
||||
var failed = false;
|
||||
string msbuild_logfile;
|
||||
try {
|
||||
AssertRunProcess ("msbuild", sb.ToArray (), TimeSpan.FromMinutes (5), Configuration.SampleRootDirectory, environment_variables, "build", out msbuild_logfile);
|
||||
} catch {
|
||||
failed = true;
|
||||
throw;
|
||||
} finally {
|
||||
watch.Stop ();
|
||||
}
|
||||
|
||||
// Write performance data to disk
|
||||
var subdirs = Directory.GetDirectories (slndir, "*", SearchOption.AllDirectories);
|
||||
|
||||
// First figure out which .app subdirectory is the actual .app. This is a bit more complicated than it would seem...
|
||||
var apps = subdirs.Where ((v) => {
|
||||
var names = v.Substring (slndir.Length).Split (Path.DirectorySeparatorChar);
|
||||
if (names.Length < 2)
|
||||
return false;
|
||||
|
||||
if (!names [names.Length - 1].EndsWith (".app", StringComparison.Ordinal))
|
||||
return false;
|
||||
|
||||
if (names.Any ((v2) => v2 == "copySceneKitAssets"))
|
||||
return false;
|
||||
|
||||
var bin_idx = Array.IndexOf (names, "bin");
|
||||
var conf_idx = Array.IndexOf (names, configuration);
|
||||
if (bin_idx < 0 || conf_idx < 0)
|
||||
return false;
|
||||
|
||||
if (bin_idx > conf_idx)
|
||||
return false;
|
||||
|
||||
if (platform.Length > 0) {
|
||||
var platform_idx = Array.IndexOf (names, platform);
|
||||
if (platform_idx < 0)
|
||||
return false;
|
||||
|
||||
if (bin_idx > platform_idx)
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}).ToArray ();
|
||||
|
||||
if (apps.Length > 1) {
|
||||
// Found more than one .app subdirectory, use additional logic to choose between them.
|
||||
var filtered_apps = apps.Where ((v) => {
|
||||
// If one .app is a subdirectory of another .app, we don't care about the former.
|
||||
if (apps.Any ((v2) => v2.Length < v.Length && v.StartsWith (v2, StringComparison.Ordinal)))
|
||||
return false;
|
||||
|
||||
// If one .app is contained within another .app, we don't care about the former.
|
||||
var vname = Path.GetFileName (v);
|
||||
var otherApps = apps.Where ((v2) => v != v2);
|
||||
if (otherApps.Any ((v2) => {
|
||||
var otherSubdirs = subdirs.Where ((v3) => v3.StartsWith (v2, StringComparison.Ordinal));
|
||||
return otherSubdirs.Any ((v3) => Path.GetFileName (v3) == vname);
|
||||
}))
|
||||
return false;
|
||||
|
||||
return true;
|
||||
}).ToArray ();
|
||||
if (apps.Length == 0)
|
||||
Assert.Fail ($"Filtered away all the .apps, from:\n\t{string.Join ("\n\t", apps)}");
|
||||
apps = filtered_apps;
|
||||
}
|
||||
|
||||
if (apps.Length > 1) {
|
||||
Assert.Fail ($"Found more than one .app directory:\n\t{string.Join ("\n\t", apps)}");
|
||||
} else if (apps.Length == 0) {
|
||||
Assert.Fail ($"Found no .app directories for platform: {platform} configuration: {configuration} target: {target}. All directories:\n\t{string.Join ("\n\t", subdirs)}");
|
||||
}
|
||||
|
||||
var logfile = Path.Combine (LogDirectory, $"{Path.GetFileNameWithoutExtension (solution)}-perfdata-{Interlocked.Increment (ref counter)}.xml");
|
||||
|
||||
var xmlSettings = new XmlWriterSettings {
|
||||
Indent = true,
|
||||
};
|
||||
var xml = XmlWriter.Create (logfile, xmlSettings);
|
||||
xml.WriteStartDocument (true);
|
||||
xml.WriteStartElement ("performance");
|
||||
xml.WriteStartElement ("sample-build");
|
||||
xml.WriteAttributeString ("mono-version", Configuration.MonoVersion);
|
||||
xml.WriteAttributeString ("os-version", Configuration.OSVersion);
|
||||
xml.WriteAttributeString ("xamarin-macios-hash", Configuration.TestedHash);
|
||||
xml.WriteAttributeString ("sample-repository", Configuration.GetCurrentRemoteUrl (slndir));
|
||||
xml.WriteAttributeString ("sample-hash", Configuration.GetCurrentHash (slndir));
|
||||
xml.WriteAttributeString ("agent-machinename", Environment.GetEnvironmentVariable ("AGENT_MACHINENAME"));
|
||||
xml.WriteAttributeString ("agent-name", Environment.GetEnvironmentVariable ("AGENT_NAME"));
|
||||
foreach (var app in apps) {
|
||||
xml.WriteStartElement ("test");
|
||||
xml.WriteAttributeString ("name", TestContext.CurrentContext.Test.FullName);
|
||||
xml.WriteAttributeString ("result", failed ? "failed" : "success");
|
||||
if (platform.Length > 0)
|
||||
xml.WriteAttributeString ("platform", platform);
|
||||
xml.WriteAttributeString ("configuration", configuration);
|
||||
if (!failed) {
|
||||
xml.WriteAttributeString ("duration", watch.ElapsedTicks.ToString ());
|
||||
xml.WriteAttributeString ("duration-formatted", watch.Elapsed.ToString ());
|
||||
|
||||
var files = Directory.GetFiles (app, "*", SearchOption.AllDirectories).OrderBy ((v) => v).ToArray ();
|
||||
var lengths = files.Select ((v) => new FileInfo (v).Length).ToArray ();
|
||||
var total_size = lengths.Sum ();
|
||||
|
||||
xml.WriteAttributeString ("total-size", total_size.ToString ());
|
||||
var appstart = Path.GetDirectoryName (app).Length;
|
||||
for (var i = 0; i < files.Length; i++) {
|
||||
xml.WriteStartElement ("file");
|
||||
xml.WriteAttributeString ("name", files [i].Substring (appstart + 1));
|
||||
xml.WriteAttributeString ("size", lengths [i].ToString ());
|
||||
xml.WriteEndElement ();
|
||||
}
|
||||
|
||||
if (File.Exists (msbuild_logfile)) {
|
||||
var lines = File.ReadAllLines (msbuild_logfile);
|
||||
var target_perf_summary = new List<string> ();
|
||||
var task_perf_summary = new List<string> ();
|
||||
var timestamps = new List<string> ();
|
||||
for (var i = lines.Length - 1; i >= 0; i--) {
|
||||
if (lines [i].EndsWith ("Target Performance Summary:", StringComparison.Ordinal)) {
|
||||
for (var k = i + 1; k < lines.Length && lines [k].EndsWith ("calls", StringComparison.Ordinal); k++) {
|
||||
target_perf_summary.Add (lines [k].Substring (18).Trim ());
|
||||
}
|
||||
} else if (lines [i].EndsWith ("Task Performance Summary:", StringComparison.Ordinal)) {
|
||||
for (var k = i + 1; k < lines.Length && lines [k].EndsWith ("calls", StringComparison.Ordinal); k++) {
|
||||
task_perf_summary.Add (lines [k].Substring (18).Trim ());
|
||||
}
|
||||
} else if (lines [i].Contains ("!Timestamp")) {
|
||||
timestamps.Add (lines [i]);
|
||||
}
|
||||
}
|
||||
foreach (var tps in target_perf_summary) {
|
||||
var split = tps.Split (new char [] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
|
||||
xml.WriteStartElement ("target");
|
||||
xml.WriteAttributeString ("name", split [2]);
|
||||
xml.WriteAttributeString ("ms", split [0]);
|
||||
xml.WriteAttributeString ("calls", split [3]);
|
||||
xml.WriteEndElement ();
|
||||
}
|
||||
foreach (var tps in task_perf_summary) {
|
||||
var split = tps.Split (new char [] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
|
||||
xml.WriteStartElement ("task");
|
||||
xml.WriteAttributeString ("name", split [2]);
|
||||
xml.WriteAttributeString ("ms", split [0]);
|
||||
xml.WriteAttributeString ("calls", split [3]);
|
||||
xml.WriteEndElement ();
|
||||
}
|
||||
foreach (var ts in timestamps) {
|
||||
// Sample line:
|
||||
// 15:04:50.4609520: !Timestamp Setup: 28 ms (TaskId:137)
|
||||
var splitFirst = ts.Split (new char [] { ':' }, StringSplitOptions.RemoveEmptyEntries);
|
||||
var splitSecondA = splitFirst [3].Split (new char [] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
|
||||
var splitSecondB = splitFirst [4].Split (new char [] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
|
||||
var name = string.Join (" ", splitSecondA.Skip (1));
|
||||
var level = splitSecondA [0].Count ((v) => v == '!').ToString ();
|
||||
var ms = splitSecondB [0];
|
||||
xml.WriteStartElement ("timestamp");
|
||||
xml.WriteAttributeString ("name", name);
|
||||
xml.WriteAttributeString ("level", level);
|
||||
xml.WriteAttributeString ("ms", ms);
|
||||
xml.WriteEndElement ();
|
||||
}
|
||||
}
|
||||
|
||||
xml.WriteEndElement ();
|
||||
}
|
||||
|
||||
xml.WriteEndElement (); // sample-build
|
||||
xml.WriteEndElement (); // performance
|
||||
xml.WriteEndDocument ();
|
||||
xml.Dispose ();
|
||||
|
||||
TestContext.AddTestAttachment (logfile, $"Performance data");
|
||||
}
|
||||
}
|
||||
|
||||
internal static string RunProcess (string filename, string arguments = "", string working_directory = null)
|
||||
{
|
||||
using (var p = Process.Start (filename, arguments)) {
|
||||
p.StartInfo.RedirectStandardOutput = true;
|
||||
p.StartInfo.UseShellExecute = false;
|
||||
if (!string.IsNullOrEmpty (working_directory))
|
||||
p.StartInfo.WorkingDirectory = working_directory;
|
||||
p.Start ();
|
||||
var output = p.StandardOutput.ReadToEnd ();
|
||||
p.WaitForExit ();
|
||||
return output;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -247,4 +247,16 @@ namespace Samples {
|
|||
return GitHub.CloneRepository (Org, Repository, Hash);
|
||||
}
|
||||
}
|
||||
|
||||
[TestFixture]
|
||||
public class BaselineTester {
|
||||
[Test]
|
||||
public void DeviceDebug ()
|
||||
{
|
||||
var sln = Path.Combine (Configuration.SourceRoot, "tests", "sampletester", "BaselineTest", "BaselineTest.sln");
|
||||
GitHub.CleanRepository (Path.GetDirectoryName (sln));
|
||||
ProcessHelper.BuildSolution (sln, "iPhone", "Debug", new Dictionary<string, string> ());
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ namespace Samples {
|
|||
const string ORG = "xamarin";
|
||||
const string REPO = "ios-samples"; // monotouch-samples redirects to ios-samples
|
||||
const string CATEGORY = "iossamples"; // categories can't contain dashes
|
||||
const string HASH = "bffa511ecb8f74b2d4a42418a130d0c83c9723cf";
|
||||
const string HASH = "150f0e6167e2133182924114a3b36e8384f632f1";
|
||||
|
||||
static Dictionary<string, SampleTest> test_data = new Dictionary<string, SampleTest> {
|
||||
// Build solution instead of csproj
|
||||
|
|
|
|||
|
|
@ -13,5 +13,13 @@ Global
|
|||
{7340A1C6-61A5-42D2-9DBC-6688D2E70F62}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{7340A1C6-61A5-42D2-9DBC-6688D2E70F62}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{7340A1C6-61A5-42D2-9DBC-6688D2E70F62}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{7340A1C6-61A5-42D2-9DBC-6688D2E70F62}.Debug|iPhoneSimulator.ActiveCfg = Debug|Any CPU
|
||||
{7340A1C6-61A5-42D2-9DBC-6688D2E70F62}.Debug|iPhoneSimulator.Build.0 = Debug|Any CPU
|
||||
{7340A1C6-61A5-42D2-9DBC-6688D2E70F62}.Release|iPhoneSimulator.ActiveCfg = Release|Any CPU
|
||||
{7340A1C6-61A5-42D2-9DBC-6688D2E70F62}.Release|iPhoneSimulator.Build.0 = Release|Any CPU
|
||||
{7340A1C6-61A5-42D2-9DBC-6688D2E70F62}.Debug|iPhone.ActiveCfg = Debug|Any CPU
|
||||
{7340A1C6-61A5-42D2-9DBC-6688D2E70F62}.Debug|iPhone.Build.0 = Debug|Any CPU
|
||||
{7340A1C6-61A5-42D2-9DBC-6688D2E70F62}.Release|iPhone.ActiveCfg = Release|Any CPU
|
||||
{7340A1C6-61A5-42D2-9DBC-6688D2E70F62}.Release|iPhone.Build.0 = Release|Any CPU
|
||||
EndGlobalSection
|
||||
EndGlobal
|
||||
|
|
|
|||
|
|
@ -1,8 +1,6 @@
|
|||
#!/bin/bash -eux
|
||||
|
||||
# we want verbose output from mtouch and mlaunch
|
||||
echo 123456789 > ~/.mtouch-verbosity
|
||||
echo 123456789 > ~/.mlaunch-verbosity
|
||||
cd "$(dirname "${BASH_SOURCE[0]}")/../.."
|
||||
|
||||
make -C tests test-system.config
|
||||
make -C tests/sampletester TESTS_USE_SYSTEM=1
|
||||
|
|
@ -13,6 +13,23 @@ variables:
|
|||
name_filter_gr: '^[G-Rg-r].*$'
|
||||
name_filter_rest: '^[^A-Ra-r].*$'
|
||||
|
||||
resources:
|
||||
repositories:
|
||||
- repository: xamarin-macios-data
|
||||
type: github
|
||||
name: xamarin/xamarin-macios-data
|
||||
ref: refs/heads/master
|
||||
endpoint: xamarin
|
||||
- repository: maccore
|
||||
type: github
|
||||
name: xamarin/maccore
|
||||
ref: refs/heads/master
|
||||
endpoint: xamarin
|
||||
|
||||
###
|
||||
### Tell GitHub we're starting working on this commit
|
||||
###
|
||||
|
||||
jobs:
|
||||
- job: ReportStartToGitHub
|
||||
displayName: Set pending GitHub status
|
||||
|
|
@ -22,6 +39,13 @@ jobs:
|
|||
- bash: ./jenkins/add-commit-status.sh "--token=$(github-pat)" "--hash=$BUILD_SOURCEVERSION" "--state=pending" --target-url="$AZURE_BUILD_URL" --description="$BUILD_DEFINITIONNAME" --context="$BUILD_DEFINITIONNAME"
|
||||
displayName: Set pending GitHub status
|
||||
|
||||
###
|
||||
### Run the sample tests.
|
||||
###
|
||||
### They're split over multiple bots to make them run faster (and not hit the
|
||||
### max job time duration for a single job).
|
||||
###
|
||||
|
||||
- job: macOS
|
||||
dependsOn: ReportStartToGitHub
|
||||
displayName: Build samples
|
||||
|
|
@ -73,32 +97,35 @@ jobs:
|
|||
|
||||
steps:
|
||||
- checkout: self
|
||||
- checkout: xamarin-macios-data
|
||||
persistCredentials: true
|
||||
- checkout: maccore
|
||||
persistCredentials: true
|
||||
|
||||
- bash: ./tools/devops/system-info.sh
|
||||
- bash: ./xamarin-macios/tools/devops/system-info.sh
|
||||
displayName: System Info
|
||||
|
||||
- bash: ./tools/devops/fetch-maccore.sh
|
||||
displayName: Fetch maccore
|
||||
- bash: ./xamarin-macios/tools/devops/fetch-maccore.sh
|
||||
displayName: Fetch correct maccore hash
|
||||
|
||||
- task: provisionator@2
|
||||
displayName: Xcode
|
||||
inputs:
|
||||
provisionator_uri: '$(provisionator-uri)'
|
||||
github_token: '$(github-pat)'
|
||||
provisioning_script: $(System.DefaultWorkingDirectory)/tools/devops/provision-xcode.csx
|
||||
provisioning_script: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/provision-xcode.csx
|
||||
|
||||
- task: provisionator@2
|
||||
displayName: Provision XI/XM/Mono/Objective-Sharpie
|
||||
inputs:
|
||||
provisionator_uri: '$(provisionator-uri)'
|
||||
github_token: '$(github-pat)'
|
||||
provisioning_script: $(System.DefaultWorkingDirectory)/tools/devops/build-samples.csx
|
||||
provisioning_script: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/build-samples.csx
|
||||
|
||||
- bash: ./tools/devops/system-info.sh
|
||||
- bash: ./xamarin-macios/tools/devops/system-info.sh
|
||||
displayName: System Info post provisioning
|
||||
|
||||
- bash: ./tools/devops/build-samples.sh
|
||||
- bash: ./xamarin-macios/tools/devops/build-samples.sh
|
||||
displayName: Run tests
|
||||
|
||||
- task: PublishTestResults@2
|
||||
|
|
@ -111,14 +138,58 @@ jobs:
|
|||
publishRunAttachments: true
|
||||
mergeTestResults: true
|
||||
|
||||
- bash: ./xamarin-macios/tools/devops/prepare-performance-data.sh
|
||||
displayName: Prepare performance data
|
||||
condition: always()
|
||||
|
||||
- publish: logs
|
||||
displayName: 'Upload perf data'
|
||||
condition: always()
|
||||
artifact: logs-$(System.JobId)
|
||||
|
||||
- bash: echo "##vso[task.setvariable variable=JobStatus;isOutput=true]$AGENT_JOBSTATUS"
|
||||
name: ExportedVariables
|
||||
displayName: Export status
|
||||
condition: always()
|
||||
|
||||
###
|
||||
### Push performance data to the xamarin-macios-data repository
|
||||
###
|
||||
|
||||
- job: PublishPerformanceData
|
||||
displayName: Publish Performance Data
|
||||
condition: always()
|
||||
dependsOn: macOS
|
||||
pool:
|
||||
name: '$(macOSVersion)'
|
||||
steps:
|
||||
- checkout: self
|
||||
- checkout: xamarin-macios-data
|
||||
persistCredentials: true
|
||||
|
||||
- task: DownloadPipelineArtifact@2
|
||||
displayName: Download performance data
|
||||
inputs:
|
||||
source: current
|
||||
path: logs
|
||||
|
||||
- bash: ./xamarin-macios/tools/devops/push-performance-data.sh
|
||||
displayName: Publish perf data
|
||||
|
||||
- bash: echo "##vso[task.setvariable variable=JobStatus;isOutput=true]$AGENT_JOBSTATUS"
|
||||
name: ExportedVariables
|
||||
displayName: Export status
|
||||
condition: always()
|
||||
|
||||
###
|
||||
### Report final results to GitHub
|
||||
###
|
||||
|
||||
- job: ReportResultsToGitHub
|
||||
displayName: Report status/results to GitHub
|
||||
dependsOn: macOS
|
||||
dependsOn:
|
||||
- PublishPerformanceData
|
||||
- macOS
|
||||
condition: always()
|
||||
pool:
|
||||
name: '$(macOSVersion)'
|
||||
|
|
@ -133,6 +204,7 @@ jobs:
|
|||
jobResultReleaseiPhoneSimulator: $[ dependencies.macOS.outputs['Release_iPhoneSimulator.ExportedVariables.JobStatus'] ]
|
||||
jobResultDebugMac: $[ dependencies.macOS.outputs['Debug_Mac.ExportedVariables.JobStatus'] ]
|
||||
jobResultReleaseMac: $[ dependencies.macOS.outputs['Release_Mac.ExportedVariables.JobStatus'] ]
|
||||
jobResultPublishPerformanceData: $[ dependencies.PublishPerformanceData.outputs['ExportedVariables.JobStatus'] ]
|
||||
steps:
|
||||
- bash: |
|
||||
./tools/devops/build-samples-report-to-github.sh "$(github-pat)" \
|
||||
|
|
@ -141,6 +213,7 @@ jobs:
|
|||
"Release_iPhone_AF" "Release_iPhone_GR" "Release_iPhone_SZ" \
|
||||
"Release_iPhoneSimulator" \
|
||||
"Debug_Mac" \
|
||||
"Release_Mac"
|
||||
"Release_Mac" \
|
||||
"PublishPerformanceData"
|
||||
displayName: Report results to GitHub as comment / status
|
||||
condition: always()
|
||||
|
|
|
|||
|
|
@ -1,12 +1,17 @@
|
|||
#!/bin/bash -eux
|
||||
|
||||
# maccore is already checked out, but our script does a different remote
|
||||
# ('xamarin' vs 'origin'), so add the different remote, and at the same time
|
||||
# use https for the repository (instead of git@), since GitHub auth on Azure
|
||||
# Devops only works with https.
|
||||
|
||||
cd "$(dirname "${BASH_SOURCE[0]}")"
|
||||
cd "$(git rev-parse --show-toplevel)/../maccore"
|
||||
git remote add -f xamarin https://github.com/xamarin/maccore
|
||||
cd ../xamarin-macios
|
||||
|
||||
# Make sure we've enabled our xamarin bits
|
||||
./configure --enable-xamarin
|
||||
|
||||
# grab Azure Devop's authorization token from the current repo, and use add it to the global git configuration
|
||||
AUTH=$(git config -l | grep AUTHORIZATION | sed 's/.*AUTHORIZATION: //')
|
||||
git config --global http.extraheader "AUTHORIZATION: $AUTH"
|
||||
|
||||
# fetch maccore
|
||||
# the github auth we use only works with https, so change maccore's url to be https:// instead of git@
|
||||
make reset-maccore MACCORE_MODULE="$(grep ^MACCORE_MODULE mk/xamarin.mk | sed -e 's/.*:= //' -e 's_git@github.com:_https://github.com/_' -e 's/[.]git//')" V=1
|
||||
# fetch the hash we want
|
||||
make reset-maccore V=1
|
||||
|
|
|
|||
|
|
@ -0,0 +1,14 @@
|
|||
#!/bin/bash -ex
|
||||
|
||||
DIR=perf-data/samples/$BUILD_SOURCEBRANCHNAME/$BUILD_SOURCEVERSION/$SYSTEM_JOBID
|
||||
mkdir -p "$DIR"
|
||||
|
||||
XMLS=(xamarin-macios/tests/sampletester/bin/Debug/tmp-test-dir/execution-logs/*.xml)
|
||||
if ! test -f "${XMLS[0]}"; then
|
||||
echo "##vso[task.logissue type=warning]Could not find any performance data to publish"
|
||||
exit 0
|
||||
fi
|
||||
cp -c xamarin-macios/tests/sampletester/bin/Debug/tmp-test-dir/execution-logs/*.xml "$DIR/"
|
||||
|
||||
mkdir -p logs
|
||||
zip -9r "logs/execution-logs-$SYSTEM_JOBID.zip" perf-data
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
#!/bin/bash -ex
|
||||
|
||||
cd xamarin-macios-data
|
||||
git checkout master
|
||||
cp -cr ../logs/ ./
|
||||
|
||||
mv ./*/*.zip .
|
||||
for zip in ./*.zip; do
|
||||
unzip "$zip"
|
||||
done
|
||||
|
||||
# Merge each individual xml file into one big xml file
|
||||
DIR=perf-data/samples/$BUILD_SOURCEBRANCHNAME/$BUILD_SOURCEVERSION
|
||||
cd "$DIR"
|
||||
# Merge the xml files from each bot into a big per-bot xml file. Don't merge
|
||||
# the xml from all the bots together into a single enormous xml file, because
|
||||
# it'll be close to GitHub's size limit per file (limit is 100mb, the enormous
|
||||
# xml file would be ~80mb now), and might very well pass that one day.
|
||||
for job in *-*-*-*-*; do
|
||||
{
|
||||
echo '<?xml version="1.0" encoding="utf-8" standalone="yes"?>'
|
||||
echo '<performance version="1.0">'
|
||||
find "$job" -name '*perfdata*.xml' -print0 | xargs -0 -n 1 tail -n +2 | grep -F -v -e '<performance>' -e '</performance>'
|
||||
echo '</performance>'
|
||||
} > "data-$job.xml"
|
||||
done
|
||||
|
||||
# Add the big xml files to git
|
||||
git add data-*.xml
|
||||
git commit -m "Add performance data for $BUILD_SOURCEBRANCHNAME/$BUILD_SOURCEVERSION."
|
||||
|
||||
# Push!
|
||||
# Try to push 5 times, just in case someone else pushed first.
|
||||
COUNTER=5
|
||||
while [[ $COUNTER -gt 0 ]]; do
|
||||
if git push; then break; fi
|
||||
git pull
|
||||
(( COUNTER-- ))
|
||||
done
|
||||
|
|
@ -10,5 +10,7 @@ ls -lad /Applications/Xcode*
|
|||
xcode-select -p
|
||||
mono --version
|
||||
env | sort
|
||||
uptime
|
||||
ps aux
|
||||
|
||||
exit 0
|
||||
exit 0
|
||||
|
|
|
|||
|
|
@ -358,6 +358,13 @@ namespace Xamarin.Bundler {
|
|||
|
||||
AddSharedOptions (App, os);
|
||||
|
||||
var extra_args = Environment.GetEnvironmentVariable ("MMP_ENV_OPTIONS");
|
||||
if (!string.IsNullOrEmpty (extra_args)) {
|
||||
var l = new List<string> (args);
|
||||
l.AddRange (extra_args.Split (new char [] { ' ' }, StringSplitOptions.RemoveEmptyEntries));
|
||||
args = l.ToArray ();
|
||||
}
|
||||
|
||||
try {
|
||||
App.RootAssemblies.AddRange (os.Parse (args));
|
||||
}
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче