microsoft
/
coyote
зеркало из
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Merged PR 1338: goto and push benchmarks 

This PR adds two new state machine benchmarks: a goto and a push one.

Also tidying up benchmarks, by moving them to a dedicated test project (until now they were inside the `BenchmarkRunner` tool, which is ugly, tests should be in the test directory).
This commit is contained in:
Pantazis Deligiannis 2020-02-27 02:45:41 +00:00
Родитель 64169331dd
Коммит 5f18ba5e9b
11 изменённых файлов: 276 добавлений и 14 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

7
Coyote.sln
Просмотреть файл

@ -38,6 +38,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CoverageReportMerger", "Too
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Coyote", "Tools\Coyote\Coyote.csproj", "{756F8C17-37FA-4A6E-8D87-3132FCE4A17D}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Performance.Tests", "Tests\Performance.Tests\Performance.Tests.csproj", "{87DD0944-2C10-46E6-9938-F7001BA1E8A7}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@ -84,6 +86,10 @@ Global
{756F8C17-37FA-4A6E-8D87-3132FCE4A17D}.Debug|Any CPU.Build.0 = Debug|Any CPU
{756F8C17-37FA-4A6E-8D87-3132FCE4A17D}.Release|Any CPU.ActiveCfg = Release|Any CPU
{756F8C17-37FA-4A6E-8D87-3132FCE4A17D}.Release|Any CPU.Build.0 = Release|Any CPU
{87DD0944-2C10-46E6-9938-F7001BA1E8A7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{87DD0944-2C10-46E6-9938-F7001BA1E8A7}.Debug|Any CPU.Build.0 = Debug|Any CPU
{87DD0944-2C10-46E6-9938-F7001BA1E8A7}.Release|Any CPU.ActiveCfg = Release|Any CPU
{87DD0944-2C10-46E6-9938-F7001BA1E8A7}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
@ -99,6 +105,7 @@ Global
{22E8C60D-916F-47C7-AF79-D1FFB0EC345F} = {9BC0914F-3068-4148-B778-4CA27D828D39}
{8A678DD5-9C51-4B71-80F7-5369CBFB8AEF} = {9BC0914F-3068-4148-B778-4CA27D828D39}
{756F8C17-37FA-4A6E-8D87-3132FCE4A17D} = {9BC0914F-3068-4148-B778-4CA27D828D39}
{87DD0944-2C10-46E6-9938-F7001BA1E8A7} = {2012300C-6E5D-47A0-9D57-B3F0A73AA1D4}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {B9407046-CB24-4B07-8031-2749696EC7D8}

12
Source/Core/Properties/InternalsVisibleTo.cs
Просмотреть файл

@ -24,12 +24,6 @@ using System.Runtime.CompilerServices;
"d7bc9fd67a08d3fa122120a469158da22a652af4508571ac9b16c6a05d2b3b6d7004ac76be85c3" +
"ca3d55f6ae823cd287a2810243f2bd6be5f4ba7b016c80da954371e591b10c97b0938f721c7149" +
"3bc97f9e")]
[assembly: InternalsVisibleTo("BenchmarkRunner,PublicKey=" +
"0024000004800000940000000602000000240000525341310004000001000100d7971281941569" +
"53fd8af100ac5ecaf1d96fab578562b91133663d6ccbf0b313d037a830a20d7af1ce02a6641d71" +
"d7bc9fd67a08d3fa122120a469158da22a652af4508571ac9b16c6a05d2b3b6d7004ac76be85c3" +
"ca3d55f6ae823cd287a2810243f2bd6be5f4ba7b016c80da954371e591b10c97b0938f721c7149" +
"3bc97f9e")]
// Tests
[assembly: InternalsVisibleTo("Microsoft.Coyote.Core.Tests,PublicKey=" +
@ -50,3 +44,9 @@ using System.Runtime.CompilerServices;
"d7bc9fd67a08d3fa122120a469158da22a652af4508571ac9b16c6a05d2b3b6d7004ac76be85c3" +
"ca3d55f6ae823cd287a2810243f2bd6be5f4ba7b016c80da954371e591b10c97b0938f721c7149" +
"3bc97f9e")]
[assembly: InternalsVisibleTo("Microsoft.Coyote.Performance.Tests,PublicKey=" +
"0024000004800000940000000602000000240000525341310004000001000100d7971281941569" +
"53fd8af100ac5ecaf1d96fab578562b91133663d6ccbf0b313d037a830a20d7af1ce02a6641d71" +
"d7bc9fd67a08d3fa122120a469158da22a652af4508571ac9b16c6a05d2b3b6d7004ac76be85c3" +
"ca3d55f6ae823cd287a2810243f2bd6be5f4ba7b016c80da954371e591b10c97b0938f721c7149" +
"3bc97f9e")]

22
Tests/Performance.Tests/Performance.Tests.csproj Normal file
Просмотреть файл

@ -0,0 +1,22 @@
<Project Sdk="Microsoft.NET.Sdk">
<Import Project="..\..\Common\build.props" />
<Import Project="..\..\Common\key.props" />
<PropertyGroup>
<Description>The Coyote performance tests.</Description>
<AssemblyName>Microsoft.Coyote.Performance.Tests</AssemblyName>
<RootNamespace>Microsoft.Coyote.Performance.Tests</RootNamespace>
<OutputPath>.\bin\</OutputPath>
</PropertyGroup>
<PropertyGroup Condition="'$(OS)'=='Windows_NT'">
<TargetFrameworks>netstandard2.0;net472</TargetFrameworks>
</PropertyGroup>
<PropertyGroup Condition="'$(OS)'=='UNIX'">
<TargetFrameworks>netstandard2.0</TargetFrameworks>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\..\Source\Core\Core.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="BenchmarkDotNet" Version="0.11.4" />
</ItemGroup>
</Project>

2
Tools/BenchmarkRunner/Tests/Actors/StateMachines/Creation/CreationThroughputBenchmark.cs → Tests/Performance.Tests/Tests/Actors/StateMachines/Creation/CreationThroughputBenchmark.cs
Просмотреть файл

@ -7,7 +7,7 @@ using BenchmarkDotNet.Attributes;
using Microsoft.Coyote.Actors;
using Microsoft.Coyote.Runtime;
namespace Microsoft.Coyote.Benchmarking.Actors.StateMachines
namespace Microsoft.Coyote.Performance.Tests.Actors.StateMachines
{
[ClrJob(baseline: true), CoreJob]
[MemoryDiagnoser]

2
Tools/BenchmarkRunner/Tests/Actors/StateMachines/Messaging/DequeueEventThroughputBenchmark.cs → Tests/Performance.Tests/Tests/Actors/StateMachines/Messaging/DequeueEventThroughputBenchmark.cs
Просмотреть файл

@ -6,7 +6,7 @@ using BenchmarkDotNet.Attributes;
using Microsoft.Coyote.Actors;
using Microsoft.Coyote.Runtime;
namespace Microsoft.Coyote.Benchmarking.Actors.StateMachines
namespace Microsoft.Coyote.Performance.Tests.Actors.StateMachines
{
[ClrJob(baseline: true), CoreJob]
[MemoryDiagnoser]

2
Tools/BenchmarkRunner/Tests/Actors/StateMachines/Messaging/ExchangeEventLatencyBenchmark.cs → Tests/Performance.Tests/Tests/Actors/StateMachines/Messaging/ExchangeEventLatencyBenchmark.cs
Просмотреть файл

@ -6,7 +6,7 @@ using BenchmarkDotNet.Attributes;
using Microsoft.Coyote.Actors;
using Microsoft.Coyote.Runtime;
namespace Microsoft.Coyote.Benchmarking.Actors.StateMachines
namespace Microsoft.Coyote.Performance.Tests.Actors.StateMachines
{
[ClrJob(baseline: true), CoreJob]
[MemoryDiagnoser]

2
Tools/BenchmarkRunner/Tests/Actors/StateMachines/Messaging/SendEventThroughputBenchmark.cs → Tests/Performance.Tests/Tests/Actors/StateMachines/Messaging/SendEventThroughputBenchmark.cs
Просмотреть файл

@ -6,7 +6,7 @@ using BenchmarkDotNet.Attributes;
using Microsoft.Coyote.Actors;
using Microsoft.Coyote.Runtime;
namespace Microsoft.Coyote.Benchmarking.Actors.StateMachines
namespace Microsoft.Coyote.Performance.Tests.Actors.StateMachines
{
[ClrJob(baseline: true), CoreJob]
[MemoryDiagnoser]

108
Tests/Performance.Tests/Tests/Actors/StateMachines/Transitions/GotoTransitionThroughputBenchmark.cs Normal file
Просмотреть файл

@ -0,0 +1,108 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using System;
using System.Threading.Tasks;
using BenchmarkDotNet.Attributes;
using Microsoft.Coyote.Actors;
using Microsoft.Coyote.Runtime;
namespace Microsoft.Coyote.Performance.Tests.Actors.StateMachines
{
[ClrJob(baseline: true), CoreJob]
[MemoryDiagnoser]
[MinColumn, MaxColumn, MeanColumn, Q1Column, Q3Column, RankColumn]
[MarkdownExporter, HtmlExporter, CsvExporter, CsvMeasurementsExporter, RPlotExporter]
public class GotoTransitionThroughputBenchmark
{
private class SetupEvent : Event
{
public TaskCompletionSource<bool> Tcs;
public int NumTransitions;
public SetupEvent(TaskCompletionSource<bool> tcs, int numTransitions)
{
this.Tcs = tcs;
this.NumTransitions = numTransitions;
}
}
private class Trigger : Event
{
public static Trigger Instance { get; } = new Trigger();
private Trigger()
{
}
}
private class M : StateMachine
{
private TaskCompletionSource<bool> Tcs;
private int NumTransitions;
[Start]
[OnEntry(nameof(InitOnEntry))]
private class Init : State
{
}
[OnEntry(nameof(PingOnEntry))]
[OnEventGotoState(typeof(Trigger), typeof(Pong))]
private class Ping : State
{
}
[OnEntry(nameof(PongOnEntry))]
private class Pong : State
{
}
private void InitOnEntry(Event e)
{
this.Tcs = (e as SetupEvent).Tcs;
this.NumTransitions = (e as SetupEvent).NumTransitions;
this.RaiseGotoStateEvent(typeof(Ping));
}
private void PingOnEntry() => this.DoTransitionFromState(typeof(Ping));
private void PongOnEntry() => this.DoTransitionFromState(typeof(Pong));
private void DoTransitionFromState(Type fromState)
{
if (this.NumTransitions > 0 && fromState == typeof(Ping))
{
this.RaiseEvent(Trigger.Instance);
}
else if (this.NumTransitions > 0 && fromState == typeof(Pong))
{
this.RaiseGotoStateEvent(typeof(Ping));
}
else if (this.NumTransitions == 0)
{
this.RaiseHaltEvent();
this.Tcs.TrySetResult(true);
}
this.NumTransitions--;
}
}
[Params(1000, 10000, 100000)]
public int NumTransitions { get; set; }
[Benchmark]
public void MeasureGotoTransitionThroughput()
{
var configuration = Configuration.Create();
var runtime = ActorRuntimeFactory.CreateProductionRuntime(configuration);
var tcs = new TaskCompletionSource<bool>();
var e = new SetupEvent(tcs, this.NumTransitions);
runtime.CreateActor(typeof(M), null, e);
tcs.Task.Wait();
}
}
}

121
Tests/Performance.Tests/Tests/Actors/StateMachines/Transitions/PushTransitionThroughputBenchmark.cs Normal file
Просмотреть файл

@ -0,0 +1,121 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using System;
using System.Threading.Tasks;
using BenchmarkDotNet.Attributes;
using Microsoft.Coyote.Actors;
using Microsoft.Coyote.Runtime;
namespace Microsoft.Coyote.Performance.Tests.Actors.StateMachines
{
[ClrJob(baseline: true), CoreJob]
[MemoryDiagnoser]
[MinColumn, MaxColumn, MeanColumn, Q1Column, Q3Column, RankColumn]
[MarkdownExporter, HtmlExporter, CsvExporter, CsvMeasurementsExporter, RPlotExporter]
public class PushTransitionThroughputBenchmark
{
private class SetupEvent : Event
{
public TaskCompletionSource<bool> Tcs;
public int NumTransitions;
public SetupEvent(TaskCompletionSource<bool> tcs, int numTransitions)
{
this.Tcs = tcs;
this.NumTransitions = numTransitions;
}
}
private class Trigger : Event
{
public static Trigger Instance { get; } = new Trigger();
private Trigger()
{
}
}
private class M : StateMachine
{
private TaskCompletionSource<bool> Tcs;
private int NumTransitions;
[Start]
[OnEntry(nameof(InitOnEntry))]
private class Init : State
{
}
[OnEntry(nameof(DoBottomTransition))]
[OnEventPushState(typeof(Trigger), typeof(Middle))]
private class Bottom : State
{
}
[OnEntry(nameof(DoMiddleTransition))]
private class Middle : State
{
}
[OnEntry(nameof(DoTopTransition))]
[OnEventDoAction(typeof(Trigger), nameof(RaisePopStateEvent))]
private class Top : State
{
}
private void InitOnEntry(Event e)
{
this.Tcs = (e as SetupEvent).Tcs;
this.NumTransitions = (e as SetupEvent).NumTransitions;
this.RaiseGotoStateEvent(typeof(Bottom));
}
private void DoBottomTransition() => this.DoTransitionFromState(typeof(Bottom));
private void DoMiddleTransition() => this.DoTransitionFromState(typeof(Middle));
private void DoTopTransition() => this.DoTransitionFromState(typeof(Top));
private void DoTransitionFromState(Type fromState)
{
if (this.NumTransitions > 0 && fromState == typeof(Bottom))
{
this.RaiseEvent(Trigger.Instance);
}
else if (this.NumTransitions > 0 && fromState == typeof(Middle))
{
this.RaisePushStateEvent(typeof(Top));
}
else if (this.NumTransitions > 0 && fromState == typeof(Top))
{
this.SendEvent(this.Id, Trigger.Instance);
this.RaisePopStateEvent();
}
else if (this.NumTransitions == 0)
{
this.RaiseHaltEvent();
this.Tcs.TrySetResult(true);
}
this.NumTransitions--;
}
}
[Params(1000, 10000, 100000)]
public int NumTransitions { get; set; }
[Benchmark]
public void MeasurePushTransitionThroughput()
{
var configuration = Configuration.Create();
var runtime = ActorRuntimeFactory.CreateProductionRuntime(configuration);
var tcs = new TaskCompletionSource<bool>();
var e = new SetupEvent(tcs, this.NumTransitions);
runtime.CreateActor(typeof(M), null, e);
tcs.Task.Wait();
}
}
}

1
Tools/BenchmarkRunner/BenchmarkRunner.csproj
Просмотреть файл

@ -16,6 +16,7 @@
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\..\Source\Core\Core.csproj" />
<ProjectReference Include="..\..\Tests\Performance.Tests\Performance.Tests.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="BenchmarkDotNet" Version="0.11.4" />

11
Tools/BenchmarkRunner/Program.cs
Просмотреть файл

@ -2,6 +2,7 @@
// Licensed under the MIT License.
using BenchmarkDotNet.Running;
using StateMachineTests = Microsoft.Coyote.Performance.Tests.Actors.StateMachines;
namespace Microsoft.Coyote.Benchmarking
{
@ -14,10 +15,12 @@ namespace Microsoft.Coyote.Benchmarking
private static void Main(string[] args)
{
// BenchmarkSwitcher.FromAssembly(typeof(Program).GetTypeInfo().Assembly).Run(args);
BenchmarkRunner.Run<Actors.StateMachines.CreationThroughputBenchmark>();
BenchmarkRunner.Run<Actors.StateMachines.ExchangeEventLatencyBenchmark>();
BenchmarkRunner.Run<Actors.StateMachines.SendEventThroughputBenchmark>();
BenchmarkRunner.Run<Actors.StateMachines.DequeueEventThroughputBenchmark>();
BenchmarkRunner.Run<StateMachineTests.CreationThroughputBenchmark>();
BenchmarkRunner.Run<StateMachineTests.ExchangeEventLatencyBenchmark>();
BenchmarkRunner.Run<StateMachineTests.SendEventThroughputBenchmark>();
BenchmarkRunner.Run<StateMachineTests.DequeueEventThroughputBenchmark>();
BenchmarkRunner.Run<StateMachineTests.GotoTransitionThroughputBenchmark>();
BenchmarkRunner.Run<StateMachineTests.PushTransitionThroughputBenchmark>();
}
#pragma warning restore CA1801 // Parameter not used
}