[generator] Add unit test project, and port a few tests over from the makefile tests. (#2299)

This commit is contained in:
Rolf Bjarne Kvinge 2017-07-10 12:46:31 +02:00 коммит произвёл GitHub
Родитель 9ed308dd11
Коммит 3d429a106b
12 изменённых файлов: 310 добавлений и 35 удалений

Просмотреть файл

@ -3,12 +3,21 @@ Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio 2012
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "generator-ikvm", "generator-ikvm.csproj", "{D2EE02C0-9BFD-477D-AC92-4DE2D8490790}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "generator-tests", "..\tests\generator\generator-tests.csproj", "{10790816-D00E-40A0-8653-2A8AB4DD33A9}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{D2EE02C0-9BFD-477D-AC92-4DE2D8490790}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{D2EE02C0-9BFD-477D-AC92-4DE2D8490790}.Debug|Any CPU.Build.0 = Debug|Any CPU
{D2EE02C0-9BFD-477D-AC92-4DE2D8490790}.Release|Any CPU.ActiveCfg = Release|Any CPU
{D2EE02C0-9BFD-477D-AC92-4DE2D8490790}.Release|Any CPU.Build.0 = Release|Any CPU
{10790816-D00E-40A0-8653-2A8AB4DD33A9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{10790816-D00E-40A0-8653-2A8AB4DD33A9}.Debug|Any CPU.Build.0 = Debug|Any CPU
{10790816-D00E-40A0-8653-2A8AB4DD33A9}.Release|Any CPU.ActiveCfg = Release|Any CPU
{10790816-D00E-40A0-8653-2A8AB4DD33A9}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
EndGlobal

Просмотреть файл

@ -325,6 +325,12 @@ namespace Xamarin.Tests
}
}
public static string BGenPath {
get {
return Path.Combine (SdkBinDir, "bgen");
}
}
public static string MmpPath {
get {
return Path.Combine (BinDirXM, "mmp");

Просмотреть файл

@ -33,7 +33,7 @@ namespace Xamarin.Tests
}
}
class Tool
abstract class Tool
{
StringBuilder output = new StringBuilder ();
@ -43,6 +43,7 @@ namespace Xamarin.Tests
public Dictionary<string, string> EnvironmentVariables { get; set; }
public TimeSpan Timeout { get; set; } = TimeSpan.FromSeconds (60);
public string WorkingDirectory;
public IEnumerable<ToolMessage> Messages { get { return messages; } }
public List<string> OutputLines {
@ -57,12 +58,12 @@ namespace Xamarin.Tests
public int Execute (string arguments, params string [] args)
{
return Execute (Configuration.MtouchPath, arguments, false, args);
return Execute (ToolPath, arguments, false, args);
}
public int Execute (string arguments, bool always_show_output, params string [] args)
{
return Execute (Configuration.MtouchPath, arguments, always_show_output, args);
return Execute (ToolPath, arguments, always_show_output, args);
}
public int Execute (string toolPath, string arguments, params string [] args)
@ -75,7 +76,7 @@ namespace Xamarin.Tests
output.Clear ();
output_lines = null;
var rv = ExecutionHelper.Execute (toolPath, string.Format (arguments, args), EnvironmentVariables, output, output);
var rv = ExecutionHelper.Execute (toolPath, string.Format (arguments, args), EnvironmentVariables, output, output, workingDirectory: WorkingDirectory);
if ((rv != 0 || always_show_output) && output.Length > 0)
Console.WriteLine ("\t" + output.ToString ().Replace ("\n", "\n\t"));
@ -114,10 +115,17 @@ namespace Xamarin.Tests
}
if (line.Length < 7)
continue; // something else
msg.Prefix = line.Substring (0, 2);
if (!int.TryParse (line.Substring (2, 4), out msg.Number))
continue; // something else
msg.Message = line.Substring (8);
line = line.Substring (8);
var toolName = MessageToolName;
if (toolName != null && line.StartsWith (toolName + ": ", StringComparison.Ordinal))
line = line.Substring (toolName.Length + 2);
msg.Message = line;
if (!string.IsNullOrEmpty (origin)) {
var idx = origin.IndexOf ('(');
@ -157,7 +165,7 @@ namespace Xamarin.Tests
public void AssertErrorPattern (int number, string messagePattern)
{
AssertErrorPattern ("MT", number, messagePattern);
AssertErrorPattern (MessagePrefix, number, messagePattern);
}
public void AssertErrorPattern (string prefix, int number, string messagePattern)
@ -174,7 +182,7 @@ namespace Xamarin.Tests
public void AssertError (int number, string message, string filename = null, int? linenumber = null)
{
AssertError ("MT", number, message, filename, linenumber);
AssertError (MessagePrefix, number, message, filename, linenumber);
}
public void AssertError (string prefix, int number, string message, string filename = null, int? linenumber = null)
@ -216,7 +224,7 @@ namespace Xamarin.Tests
public void AssertWarningPattern (int number, string messagePattern)
{
AssertWarningPattern ("MT", number, messagePattern);
AssertWarningPattern (MessagePrefix, number, messagePattern);
}
public void AssertWarningPattern (string prefix, int number, string messagePattern)
@ -233,7 +241,7 @@ namespace Xamarin.Tests
public void AssertWarning (int number, string message)
{
AssertWarning ("MT", number, message);
AssertWarning (MessagePrefix, number, message);
}
public void AssertWarning (string prefix, int number, string message)
@ -283,6 +291,10 @@ namespace Xamarin.Tests
foreach (var line in OutputLines)
action (line);
}
protected abstract string ToolPath { get; }
protected abstract string MessagePrefix { get; }
protected virtual string MessageToolName { get { return null; } }
}
class XBuild
@ -380,7 +392,7 @@ namespace Xamarin.Tests
return rv;
}
public static int Execute (string fileName, string arguments, Dictionary<string, string> environmentVariables, StringBuilder stdout, StringBuilder stderr, TimeSpan? timeout = null)
public static int Execute (string fileName, string arguments, Dictionary<string, string> environmentVariables, StringBuilder stdout, StringBuilder stderr, TimeSpan? timeout = null, string workingDirectory = null)
{
if (stdout == null)
stdout = new StringBuilder ();
@ -390,6 +402,8 @@ namespace Xamarin.Tests
var psi = new ProcessStartInfo ();
psi.FileName = fileName;
psi.Arguments = arguments;
if (!string.IsNullOrEmpty (workingDirectory))
psi.WorkingDirectory = workingDirectory;
if (environmentVariables != null) {
var envs = psi.EnvironmentVariables;
foreach (var kvp in environmentVariables) {

104
tests/generator/BGenTool.cs Normal file
Просмотреть файл

@ -0,0 +1,104 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
using NUnit.Framework;
using Xamarin.Utils;
namespace Xamarin.Tests
{
public enum Profile {
None,
iOS,
tvOS,
watchOS,
macClassic,
macModern,
macFull,
macSystem,
}
class BGenTool : Tool
{
public Profile Profile;
public List<string> ApiDefinitions = new List<string> ();
public string TmpDirectory;
protected override string ToolPath { get { return Configuration.BGenPath; } }
protected override string MessagePrefix { get { return "BI"; } }
protected override string MessageToolName { get { return "bgen"; } }
string BuildArguments ()
{
var sb = new StringBuilder ();
var targetFramework = (string) null;
switch (Profile) {
case Profile.None:
break;
case Profile.iOS:
targetFramework = "Xamarin.iOS,v1.0";
break;
case Profile.tvOS:
targetFramework = "Xamarin.TVOS,v1.0";
break;
case Profile.watchOS:
targetFramework = "Xamarin.WatchOS,v1.0";
break;
case Profile.macClassic:
targetFramework = "XamMac,v1.0";
break;
case Profile.macFull:
targetFramework = "Xamarin.Mac,Version=v4.5,Profile=Full";
break;
case Profile.macModern:
targetFramework = "Xamarin.Mac,Version=v2.0,Profile=Mobile";
break;
case Profile.macSystem:
targetFramework = "Xamarin.Mac,Version=v4.5,Profile=System";
break;
default:
throw new NotImplementedException ($"Profile: {Profile}");
}
if (!string.IsNullOrEmpty (targetFramework))
sb.Append (" --target-framework=").Append (targetFramework);
foreach (var ad in ApiDefinitions)
sb.Append (" --api=").Append (StringUtils.Quote (ad));
if (!string.IsNullOrEmpty (TmpDirectory))
sb.Append (" --tmpdir=").Append (StringUtils.Quote (TmpDirectory));
return sb.ToString ();
}
public void AssertExecute (string message)
{
Assert.AreEqual (0, Execute (BuildArguments ()), message);
}
public void AssertExecuteError (string message)
{
Assert.AreNotEqual (0, Execute (BuildArguments ()), message);
}
void EnsureTempDir ()
{
if (TmpDirectory == null)
TmpDirectory = Cache.CreateTemporaryDirectory ();
}
public void CreateTemporaryBinding (string api_definition)
{
EnsureTempDir ();
var api = Path.Combine (TmpDirectory, "api.cs");
File.WriteAllText (api, api_definition);
ApiDefinitions.Add (api);
WorkingDirectory = TmpDirectory;
}
}
}

Просмотреть файл

@ -0,0 +1,50 @@
using System;
using NUnit.Framework;
using Xamarin.Tests;
namespace GeneratorTests
{
[TestFixture ()]
[Parallelizable (ParallelScope.All)]
public class ErrorTests
{
[Test]
public void BI0086 ()
{
var bgen = new BGenTool ();
bgen.CreateTemporaryBinding ("");
bgen.AssertExecuteError ("build");
bgen.AssertError (86, "A target framework (--target-framework) must be specified.");
}
[Test]
public void BI1061 ()
{
var bgen = new BGenTool ();
bgen.Profile = Profile.iOS;
bgen.CreateTemporaryBinding (@"using System;
using Foundation;
namespace Bug52570Tests {
[Category]
[BaseType (typeof (FooObject))]
interface FooObject_Extensions {
[Static]
[Export (""someMethod:"")]
bool SomeMethod (NSRange range);
}
[BaseType (typeof (NSObject))]
interface FooObject {
}
}");
bgen.AssertExecute ("build");
bgen.AssertWarning (1117, "The SomeMethod member is decorated with [Static] and its container class Bug52570Tests.FooObject_Extensions is decorated with [Category] this leads to hard to use code. Please inline SomeMethod into Bug52570Tests.FooObject class.");
}
}
}

Просмотреть файл

@ -21,7 +21,7 @@ else
IOS_GENERATOR = $(IOS_CURRENT_DIR)/bin/btouch-native /baselib:$(IOS_CURRENT_DIR)/lib/mono/Xamarin.iOS/Xamarin.iOS.dll /unsafe
endif
IOS_TESTS = bug15283 bug15307 bug15799 bug16036 sof20696157 bug23041 bug27430 bug27428 bug34042 btouch-with-hyphen-in-name property-redefination-ios arrayfromhandlebug bug36457 bug39614 bug40282 bug17232 bug24078-ignore-methods-events strong-dict-support-templated-dicts bug43579 bindastests
IOS_CUSTOM_TESTS = forum54078 desk63279 desk79124 multiple-api-definitions1 multiple-api-definitions2 bug29493 classNameCollision bi1036 bug37527 bug27986 bug35176 bi1046 bindas1048error bindas1049error bindas1050modelerror bindas1050protocolerror virtualwrap bug42855 bug52570 bug52570classinternal bug52570methodinternal bug52570allowstaticmembers bug42742 warnaserror nowarn noasyncinternalwrapper noasyncwarningcs0219 bug53076 bug53076withmodel bug57070
IOS_CUSTOM_TESTS = forum54078 desk63279 desk79124 multiple-api-definitions1 multiple-api-definitions2 bug29493 classNameCollision bi1036 bug37527 bug27986 bug35176 bi1046 bindas1048error bindas1049error bindas1050modelerror bindas1050protocolerror virtualwrap bug42855 bug52570classinternal bug52570methodinternal bug52570allowstaticmembers bug42742 warnaserror nowarn noasyncinternalwrapper noasyncwarningcs0219 bug53076 bug53076withmodel bug57070
MAC_CURRENT_DIR=$(MAC_DESTDIR)/Library/Frameworks/Xamarin.Mac.framework/Versions/Current
ifdef IKVM
@ -46,8 +46,21 @@ ifdef INCLUDE_MAC
ALL_TESTS += $(MAC_TESTS) $(MAC_CUSTOM_TESTS)
endif
all-local:: $(ALL_TESTS)
all-local:: $(ALL_TESTS) run-unit-tests
build-unit-tests:
$(Q) $(SYSTEM_MONO) /Library/Frameworks//Mono.framework/Versions/Current/lib/mono/nuget/NuGet.exe restore $(TOP)/src/generator-ikvm.sln
$(SYSTEM_XBUILD) generator-tests.csproj $(XBUILD_VERBOSITY)
run-unit-tests: build-unit-tests
rm -f .failed-stamp
$(SYSTEM_MONO) --debug $(TOP)/packages/NUnit.ConsoleRunner.3.5.0/tools/nunit3-console.exe $(abspath $(TOP)/tests/generator/bin/Debug/generator-tests.dll) "--result=$(abspath $(CURDIR)/TestResult.xml);format=nunit2" $(TEST_FIXTURE) --labels=All || touch $(CURDIR)/.failed-stamp
@# Create an html file and tell MonkeyWrench to upload it (if we're running there)
@[[ -z "$$BUILD_REPOSITORY" ]] || \
( xsltproc $(TOP)/tests/HtmlTransform.xslt TestResult.xml > index.html && \
echo "@MonkeyWrench: AddFile: $$PWD/index.html")
@[[ ! -e .failed-stamp ]]
$(MAC_TESTS):
$(if $(V),,@echo "$@";) $(MAC_CLASSIC_GENERATOR) -d=MONOMAC $@.cs
$(if $(V),,@echo "$@";) $(MAC_FULL_GENERATOR) -d=MONOMAC -d=XAMCORE_2_0 $@.cs
@ -79,7 +92,6 @@ $(eval $(call iOSErrorCodeTestTemplate,bindas1049error,1049))
$(eval $(call iOSErrorCodeTestTemplate,bindas1050modelerror,1050))
$(eval $(call iOSErrorCodeTestTemplate,bindas1050protocolerror,1050))
$(eval $(call iOSErrorCodeTestTemplate,bug42855,1060))
$(eval $(call iOSErrorCodeTestTemplate,bug52570,1117))
$(eval $(call iOSErrorCodeTestTemplate,bug57070,1061))
define iOSNoErrorCodesTestTemplate

Просмотреть файл

@ -1,19 +0,0 @@
using System;
using Foundation;
namespace Bug52570Tests {
[Category]
[BaseType (typeof (FooObject))]
interface FooObject_Extensions {
[Static]
[Export ("someMethod:")]
bool SomeMethod (NSRange range);
}
[BaseType (typeof (NSObject))]
interface FooObject {
}
}

Просмотреть файл

@ -0,0 +1,54 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<ProjectGuid>{10790816-D00E-40A0-8653-2A8AB4DD33A9}</ProjectGuid>
<OutputType>Library</OutputType>
<RootNamespace>generatortests</RootNamespace>
<AssemblyName>generator-tests</AssemblyName>
<TargetFrameworkVersion>v4.6.1</TargetFrameworkVersion>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>bin\Debug</OutputPath>
<DefineConstants>DEBUG;</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<Optimize>true</Optimize>
<OutputPath>bin\Release</OutputPath>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<ItemGroup>
<Reference Include="System" />
<Reference Include="System.Xml" />
<Reference Include="nunit.framework">
<HintPath>..\..\packages\NUnit.3.7.1\lib\net45\nunit.framework.dll</HintPath>
</Reference>
</ItemGroup>
<ItemGroup>
<Compile Include="ErrorTests.cs" />
<Compile Include="BGenTool.cs" />
<Compile Include="..\common\ExecutionHelper.cs">
<Link>ExecutionHelper.cs</Link>
</Compile>
<Compile Include="..\common\Configuration.cs">
<Link>Configuration.cs</Link>
</Compile>
<Compile Include="..\mtouch\Cache.cs">
<Link>Cache.cs</Link>
</Compile>
<Compile Include="..\..\tools\common\StringUtils.cs">
<Link>StringUtils.cs</Link>
</Compile>
</ItemGroup>
<ItemGroup>
<None Include="packages.config" />
</ItemGroup>
<Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
</Project>

Просмотреть файл

@ -0,0 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="NUnit" version="3.7.1" targetFramework="net461" />
<package id="NUnit.ConsoleRunner" version="3.5.0" targetFramework="net40" />
<package id="NUnit.Extension.NUnitV2ResultWriter" version="3.5.0" targetFramework="net461" />
</packages>

Просмотреть файл

@ -98,5 +98,13 @@ namespace Xamarin
return sb.ToString ();
}
protected override string ToolPath {
get { return Configuration.MlaunchPath; }
}
protected override string MessagePrefix {
get { return "HT"; }
}
}
}

Просмотреть файл

@ -888,5 +888,13 @@ public partial class NotificationController : WKUserNotificationInterfaceControl
return ExecutionHelper.Execute ("nm", $"-gUj {StringUtils.Quote (NativeExecutablePath)}", hide_output: true).Split ('\n');
}
}
protected override string ToolPath {
get { return Configuration.MtouchPath; }
}
protected override string MessagePrefix {
get { return "MT"; }
}
}
}

Просмотреть файл

@ -334,6 +334,7 @@ namespace xharness
"src/generator.cs",
"src/generator-enums.cs",
"src/generator-filters.cs",
"tests/generator",
};
var mac_binding_project = new string [] {
"msbuild",
@ -557,12 +558,32 @@ namespace xharness
Jenkins = this,
Platform = TestPlatform.iOS,
TestName = "BTouch tests",
Target = "wrench-btouch",
WorkingDirectory = Harness.RootDirectory,
Target = "all",
WorkingDirectory = Path.Combine (Harness.RootDirectory, "generator"),
Ignored = !IncludeBtouch,
};
Tasks.Add (runBTouch);
var buildGenerator = new MakeTask {
Jenkins = this,
TestProject = new TestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "..", "src", "generator-ikvm.sln"))),
SpecifyPlatform = false,
SpecifyConfiguration = false,
Platform = TestPlatform.iOS,
Target = "build-unit-tests",
WorkingDirectory = Path.GetFullPath (Path.Combine (Harness.RootDirectory, "generator")),
};
var runGenerator = new NUnitExecuteTask (buildGenerator) {
TestLibrary = Path.Combine (Harness.RootDirectory, "generator", "bin", "Debug", "generator-tests.dll"),
TestExecutable = Path.Combine (Harness.RootDirectory, "..", "packages", "NUnit.ConsoleRunner.3.5.0", "tools", "nunit3-console.exe"),
WorkingDirectory = Path.Combine (Harness.RootDirectory, "generator", "bin", "Debug"),
Platform = TestPlatform.iOS,
TestName = "Generator tests",
Timeout = TimeSpan.FromMinutes (10),
Ignored = !IncludeBtouch,
};
Tasks.Add (runGenerator);
var run_mmp = new MakeTask
{
Jenkins = this,
@ -2278,12 +2299,14 @@ function oninitialload ()
ExecutionResult = TestExecutingResult.Running;
var result = await proc.RunAsync (log, true, Timeout);
if (result.TimedOut) {
log.WriteLine ("Execution timed out after {0} minutes.", Timeout.Minutes);
FailureMessage = $"Execution timed out after {Timeout.Minutes} minutes.";
log.WriteLine (FailureMessage);
ExecutionResult = TestExecutingResult.TimedOut;
} else if (result.Succeeded) {
ExecutionResult = TestExecutingResult.Succeeded;
} else {
ExecutionResult = TestExecutingResult.Failed;
FailureMessage = $"Execution failed with exit code {result.ExitCode}";
}
}
Jenkins.MainLog.WriteLine ("Executed {0} ({1})", TestName, Mode);