Fix Blazor WebAssembly perf benchmarks (#56811)

This commit is contained in:
Mackinnon Buck 2024-07-26 09:03:33 -07:00 коммит произвёл GitHub
Родитель 67feeaa5fc
Коммит 62ece1b90f
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: B5690EEEBB952194
11 изменённых файлов: 56 добавлений и 241 удалений

Просмотреть файл

@ -310,7 +310,7 @@
<NewtonsoftJsonVersion>13.0.3</NewtonsoftJsonVersion>
<NSwagApiDescriptionClientVersion>13.0.4</NSwagApiDescriptionClientVersion>
<PhotinoNETVersion>2.5.2</PhotinoNETVersion>
<MicrosoftPlaywrightVersion>1.28.0</MicrosoftPlaywrightVersion>
<MicrosoftPlaywrightVersion>1.45.1</MicrosoftPlaywrightVersion>
<PollyExtensionsHttpVersion>3.0.0</PollyExtensionsHttpVersion>
<PollyVersion>7.2.4</PollyVersion>
<SeleniumSupportVersion>4.22.0</SeleniumSupportVersion>

Просмотреть файл

@ -8,13 +8,14 @@ using System.Text.Encodings.Web;
using System.Text.Json;
using Microsoft.AspNetCore.Hosting.Server;
using Microsoft.AspNetCore.Hosting.Server.Features;
using OpenQA.Selenium;
using DevHostServerProgram = Microsoft.AspNetCore.Components.WebAssembly.DevServer.Server.Program;
using Microsoft.Playwright;
namespace Wasm.Performance.Driver;
public class Program
{
private const bool RunHeadless = true;
internal static TaskCompletionSource<BenchmarkResult> BenchmarkResultTask;
public static async Task<int> Main(string[] args)
@ -51,7 +52,18 @@ public class Program
// This write is required for the benchmarking infrastructure.
Console.WriteLine("Application started.");
using var browser = await Selenium.CreateBrowser(default, captureBrowserMemory: isStressRun);
var browserArgs = new List<string>();
if (isStressRun)
{
browserArgs.Add("--enable-precise-memory-info");
}
using var playwright = await Playwright.CreateAsync();
await using var browser = await playwright.Chromium.LaunchAsync(new()
{
Headless = RunHeadless,
Args = browserArgs,
});
using var testApp = StartTestApp();
using var benchmarkReceiver = StartBenchmarkResultReceiver();
var testAppUrl = GetListeningUrl(testApp);
@ -67,19 +79,20 @@ public class Program
var timeForEachRun = TimeSpan.FromMinutes(3);
var launchUrl = $"{testAppUrl}?resultsUrl={UrlEncoder.Default.Encode(receiverUrl)}#automated";
browser.Url = launchUrl;
browser.Navigate();
var page = await browser.NewPageAsync();
await page.GotoAsync(launchUrl);
page.Console += WriteBrowserConsoleMessage;
do
{
BenchmarkResultTask = new TaskCompletionSource<BenchmarkResult>();
using var runCancellationToken = new CancellationTokenSource(timeForEachRun);
using var registration = runCancellationToken.Token.Register(() =>
using var registration = runCancellationToken.Token.Register(async () =>
{
string exceptionMessage = $"Timed out after {timeForEachRun}.";
var exceptionMessage = $"Timed out after {timeForEachRun}.";
try
{
var innerHtml = browser.FindElement(By.CssSelector(":first-child")).GetAttribute("innerHTML");
var innerHtml = await page.GetAttributeAsync(":first-child", "innerHTML");
exceptionMessage += Environment.NewLine + "Browser state: " + Environment.NewLine + innerHtml;
}
catch
@ -107,6 +120,11 @@ public class Program
return 0;
}
private static void WriteBrowserConsoleMessage(object sender, IConsoleMessage message)
{
Console.WriteLine($"[Browser Log]: {message.Text}");
}
private static void FormatAsBenchmarksOutput(BenchmarkResult benchmarkResult, bool includeMetadata, bool isStressRun)
{
// Sample of the the format: https://github.com/aspnet/Benchmarks/blob/e55f9e0312a7dd019d1268c1a547d1863f0c7237/src/Benchmarks/Program.cs#L51-L67
@ -252,31 +270,20 @@ public class Program
}
}
static IHost StartTestApp()
static WebApplication StartTestApp()
{
var args = new[]
{
"--urls", "http://127.0.0.1:0",
"--applicationpath", typeof(TestApp.Program).Assembly.Location,
#if DEBUG
"--contentroot",
Path.GetFullPath(typeof(Program).Assembly.GetCustomAttributes<AssemblyMetadataAttribute>()
.First(f => f.Key == "TestAppLocatiion")
.Value)
#endif
};
string[] args = ["--urls", "http://127.0.0.1:0"];
var app = WebApplication.Create(args);
app.MapStaticAssets();
app.MapFallbackToFile("index.html");
var host = DevHostServerProgram.BuildWebHost(args);
RunInBackgroundThread(host.Start);
return host;
RunInBackgroundThread(app.Start);
return app;
}
static IHost StartBenchmarkResultReceiver()
{
var args = new[]
{
"--urls", "http://127.0.0.1:0",
};
string[] args = ["--urls", "http://127.0.0.1:0"];
var host = Host.CreateDefaultBuilder(args)
.ConfigureWebHostDefaults(builder => builder.UseStartup<BenchmarkDriverStartup>())

Просмотреть файл

@ -1,136 +0,0 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
using System.Net.Http;
using OpenQA.Selenium;
using OpenQA.Selenium.Chrome;
using OpenQA.Selenium.Remote;
namespace Wasm.Performance.Driver;
internal sealed class Selenium
{
const string SeleniumHost = "127.0.0.1";
const int SeleniumPort = 4444;
const bool RunHeadlessBrowser = true;
const bool PoolForBrowserLogs = true;
private static async ValueTask<Uri> WaitForServerAsync(string host, int port, CancellationToken cancellationToken)
{
var uri = new UriBuilder("http", host, port, "/wd/hub/").Uri;
var httpClient = new HttpClient
{
BaseAddress = uri,
Timeout = TimeSpan.FromSeconds(1),
};
Console.WriteLine($"Attempting to connect to Selenium Server running at {uri}");
const int MaxRetries = 30;
var retries = 0;
while (retries < MaxRetries)
{
retries++;
try
{
var response = (await httpClient.GetAsync("status", cancellationToken)).EnsureSuccessStatusCode();
Console.WriteLine("Connected to Selenium");
return uri;
}
catch
{
if (retries == 1)
{
Console.WriteLine("Could not connect to selenium-server. Has it been started as yet?");
}
}
await Task.Delay(1000, cancellationToken);
}
throw new Exception($"Unable to connect to selenium-server at {uri}");
}
public static async Task<RemoteWebDriver> CreateBrowser(CancellationToken cancellationToken, bool captureBrowserMemory = false)
{
var uri = await WaitForServerAsync(SeleniumHost, SeleniumPort, cancellationToken);
var options = new ChromeOptions();
if (RunHeadlessBrowser)
{
options.AddArgument("--headless");
}
if (captureBrowserMemory)
{
options.AddArgument("--enable-precise-memory-info");
}
// Chrome fails to load site resources if it fills up the /dev/shm partition,
// so we add this argument to force using a temporary directory for shared memory files.
options.AddArgument("--disable-dev-shm-usage");
options.SetLoggingPreference(LogType.Browser, OpenQA.Selenium.LogLevel.All);
var attempt = 0;
const int MaxAttempts = 3;
do
{
try
{
// The driver opens the browser window and tries to connect to it on the constructor.
// Under heavy load, this can cause issues
// To prevent this we let the client attempt several times to connect to the server, increasing
// the max allowed timeout for a command on each attempt linearly.
var driver = new CustomRemoteWebDriver(
uri,
options.ToCapabilities(),
TimeSpan.FromSeconds(60).Add(TimeSpan.FromSeconds(attempt * 60)));
driver.Manage().Timeouts().ImplicitWait = TimeSpan.FromSeconds(1);
if (PoolForBrowserLogs)
{
// Run in background.
var logs = driver.Manage().Logs;
_ = Task.Run(async () =>
{
while (!cancellationToken.IsCancellationRequested)
{
await Task.Delay(TimeSpan.FromSeconds(3));
var consoleLogs = logs.GetLog(LogType.Browser);
foreach (var entry in consoleLogs)
{
Console.WriteLine($"[Browser Log]: {entry.Timestamp}: {entry.Message}");
}
}
});
}
return driver;
}
catch (Exception ex)
{
Console.WriteLine($"Error initializing RemoteWebDriver: {ex.Message}");
}
attempt++;
} while (attempt < MaxAttempts);
throw new InvalidOperationException("Couldn't create a Selenium remote driver client. The server is irresponsive");
}
// The WebDriver must implement ISupportsLogs to enable reading browser console logs.
private sealed class CustomRemoteWebDriver : RemoteWebDriver, ISupportsLogs
{
public CustomRemoteWebDriver(Uri remoteAddress, ICapabilities desiredCapabilities, TimeSpan commandTimeout)
: base(remoteAddress, desiredCapabilities, commandTimeout)
{
}
}
}

Просмотреть файл

@ -8,8 +8,6 @@
<!-- WebDriver is not strong-named, so this test project cannot be strong named either. -->
<SignAssembly>false</SignAssembly>
<IsTestAssetProject>true</IsTestAssetProject>
<RuntimeIdentifier Condition=" '$(DotNetBuildSourceOnly)' != 'true' ">linux-x64</RuntimeIdentifier>
<SelfContained Condition=" '$(DotNetBuildSourceOnly)' != 'true' ">true</SelfContained>
<Nullable>annotations</Nullable>
</PropertyGroup>
@ -17,23 +15,9 @@
<Reference Include="Microsoft.AspNetCore" />
<Reference Include="Microsoft.AspNetCore.Cors" />
<Reference Include="Microsoft.AspNetCore.Components.WebAssembly.Server" />
<Reference Include="Selenium.Support" />
<Reference Include="Selenium.WebDriver" />
<Reference Include="Microsoft.Playwright" Condition="'$(IsPlaywrightAvailable)' == 'true'" />
<Reference Include="Microsoft.Playwright" ExcludeAssets="build" Condition="'$(IsPlaywrightAvailable)' != 'true'" />
<ProjectReference Include="..\TestApp\Wasm.Performance.TestApp.csproj" />
</ItemGroup>
<ItemGroup>
<Compile Include="..\..\..\WebAssembly\DevServer\src\Server\*.cs" />
</ItemGroup>
<Target Name="_AddTestProjectMetadataAttributes" BeforeTargets="BeforeCompile">
<ItemGroup>
<AssemblyAttribute
Include="System.Reflection.AssemblyMetadataAttribute">
<_Parameter1>TestAppLocatiion</_Parameter1>
<_Parameter2>$(MSBuildThisFileDirectory)..\TestApp\</_Parameter2>
</AssemblyAttribute>
</ItemGroup>
</Target>
</Project>

Просмотреть файл

@ -5,22 +5,16 @@ See https://github.com/aspnet/Benchmarks#benchmarks for usage guidance on using
### Running the benchmarks
The TestApp is a regular BlazorWASM project and can be run using `dotnet run`. The Driver is an app that connects against an existing Selenium server, and speaks the Benchmark protocol. You generally do not need to run the Driver locally, but if you were to do so, you can either start a selenium-server instance and run using `dotnet run [<selenium-server-port>]` or run it inside a Linux-based docker container.
The TestApp is a regular BlazorWASM project and can be run using `dotnet run`. The Driver is an app that uses Playwright to launch a browser and run the test app, reporting benchmark results in Crank's protocol format. You generally do not need to run the Driver locally, you can do so if needed via `dotnet run`.
Here are the commands you would need to run it locally inside docker:
1. `dotnet publish -c Release Driver/Wasm.Performance.Driver.csproj`
2. `docker build -t blazor-local -f ./local.dockerfile . `
3. `docker run -it blazor-local`
To run the benchmark app in the Benchmark server, run
The benchmark app can also be run using [Crank](https://github.com/dotnet/crank?tab=readme-ov-file). To run the benchmark app in the Benchmark server, follow the Crank installation steps and then run:
```
dotnet run -- --config aspnetcore/src/Components/benchmarkapps/Wasm.Performance/benchmarks.compose.json application.endpoints <BenchmarkServerUri> --scenario blazorwasmbenchmark
crank --config https://github.com/dotnet/aspnetcore/blob/main/src/Components/benchmarkapps/Wasm.Performance/benchmarks.compose.json?raw=true --config https://github.com/aspnet/Benchmarks/blob/main/scenarios/aspnet.profiles.yml?raw=true --scenario blazorwasmbenchmark --profile aspnet-perf-lin
```
If you have local changes that you'd like to benchmark, the easiest way is to push your local changes and tell the server to use your branch:
```
dotnet run -- --config aspnetcore/src/Components/benchmarkapps/Wasm.Performance/benchmarks.compose.json application.endpoints <BenchmarkServerUri> --scenario blazorwasmbenchmark --application.buildArguments "gitBranch=mylocalchanges"
crank --config https://github.com/dotnet/aspnetcore/blob/main/src/Components/benchmarkapps/Wasm.Performance/benchmarks.compose.json?raw=true --config https://github.com/aspnet/Benchmarks/blob/main/scenarios/aspnet.profiles.yml?raw=true --scenario blazorwasmbenchmark --profile aspnet-perf-lin --application.buildArguments gitBranch=myLocalChanges --application.source.branchOrCommit myLocalChanges
```

Просмотреть файл

@ -5,7 +5,7 @@
<IsTestAssetProject>true</IsTestAssetProject>
<!--
Chrome in docker appears to run in to cache corruption issues when the cache is read multiple times over.
Clien caching isn't part of our performance measurement, so we'll skip it.
Client caching isn't part of our performance measurement, so we'll skip it.
-->
<BlazorCacheBootResources>false</BlazorCacheBootResources>
</PropertyGroup>

Просмотреть файл

@ -25,22 +25,12 @@ RUN git init \
RUN ./restore.sh
RUN npm run build
RUN .dotnet/dotnet publish -c Release --no-restore -o /app ./src/Components/benchmarkapps/Wasm.Performance/Driver/Wasm.Performance.Driver.csproj
RUN .dotnet/dotnet publish -c Release -r linux-x64 --sc true -o /app ./src/Components/benchmarkapps/Wasm.Performance/Driver/Wasm.Performance.Driver.csproj
RUN chmod +x /app/Wasm.Performance.Driver
WORKDIR /app
# NOTE: This has been commented out because it is causing our builds to get a build warning
# because we are pulling this container image from docker.io. We should consider whether
# we need this code in our repo at all, and if not remove it.
# If we do need it then we need to get the container image imported into mcr.microsoft.com
#
# I have opened up a PR to do this, however it is not certain we'll be allowed to do this
# and there is further legal/compliance work that needs to be done. In the meantime commenting
# this out should get our builds to green again whilst this issue is resolved.
#
# PR: https://github.com/microsoft/mcr/pull/3232
#
# FROM selenium/standalone-chrome:124.0 as final
FROM mcr.microsoft.com/playwright/dotnet:v1.45.1-jammy-amd64 AS final
COPY --from=build ./app ./
COPY ./exec.sh ./

Просмотреть файл

@ -1,5 +1,3 @@
#!/usr/bin/env bash
/opt/bin/start-selenium-standalone.sh&
./Wasm.Performance.Driver $StressRunDuration

Просмотреть файл

@ -1,20 +0,0 @@
# NOTE: This has been commented out because it is causing our builds to get a build warning
# because we are pulling this container image from docker.io. We should consider whether
# we need this code in our repo at all, and if not remove it.
# If we do need it then we need to get the container image imported into mcr.microsoft.com
#
# I have opened up a PR to do this, however it is not certain we'll be allowed to do this
# and there is further legal/compliance work that needs to be done. In the meantime commenting
# this out should get our builds to green again whilst this issue is resolved.
#
# PR: https://github.com/microsoft/mcr/pull/3232
#
# FROM selenium/standalone-chrome:latest as final
ENV StressRunDuration=0
WORKDIR /app
COPY ./Driver/bin/Release/net9.0/linux-x64/publish ./
COPY ./exec.sh ./
ENTRYPOINT [ "bash", "./exec.sh" ]

Просмотреть файл

@ -125,7 +125,7 @@ public abstract class BlazorTemplateTest : BrowserTestBase
{
// Can navigate to the counter page
await Task.WhenAll(
page.WaitForNavigationAsync(new() { UrlString = "**/counter" }),
page.WaitForURLAsync("**/counter"),
page.WaitForSelectorAsync("h1 >> text=Counter"),
page.WaitForSelectorAsync("p >> text=Current count: 0"),
page.ClickAsync("a[href=counter]"));
@ -137,36 +137,36 @@ public abstract class BlazorTemplateTest : BrowserTestBase
if (usesAuth)
{
await Task.WhenAll(
page.WaitForNavigationAsync(new() { UrlString = "**/Identity/Account/Login**", WaitUntil = WaitUntilState.NetworkIdle }),
page.WaitForURLAsync("**/Identity/Account/Login**", new() { WaitUntil = WaitUntilState.NetworkIdle }),
page.ClickAsync("text=Log in"));
await Task.WhenAll(
page.WaitForSelectorAsync("[name=\"Input.Email\"]"),
page.WaitForNavigationAsync(new() { UrlString = "**/Identity/Account/Register**", WaitUntil = WaitUntilState.NetworkIdle }),
page.WaitForURLAsync("**/Identity/Account/Register**", new() { WaitUntil = WaitUntilState.NetworkIdle }),
page.ClickAsync("text=Register as a new user"));
var userName = $"{Guid.NewGuid()}@example.com";
var password = "[PLACEHOLDER]-1a";
await page.TypeAsync("[name=\"Input.Email\"]", userName);
await page.TypeAsync("[name=\"Input.Password\"]", password);
await page.TypeAsync("[name=\"Input.ConfirmPassword\"]", password);
await page.FillAsync("[name=\"Input.Email\"]", userName);
await page.FillAsync("[name=\"Input.Password\"]", password);
await page.FillAsync("[name=\"Input.ConfirmPassword\"]", password);
// We will be redirected to the RegisterConfirmation
await Task.WhenAll(
page.WaitForNavigationAsync(new() { UrlString = "**/Identity/Account/RegisterConfirmation**", WaitUntil = WaitUntilState.NetworkIdle }),
page.WaitForURLAsync("**/Identity/Account/RegisterConfirmation**", new() { WaitUntil = WaitUntilState.NetworkIdle }),
page.ClickAsync("#registerSubmit"));
// We will be redirected to the ConfirmEmail
await Task.WhenAll(
page.WaitForNavigationAsync(new() { UrlString = "**/Identity/Account/ConfirmEmail**", WaitUntil = WaitUntilState.NetworkIdle }),
page.WaitForURLAsync("**/Identity/Account/ConfirmEmail**", new() { WaitUntil = WaitUntilState.NetworkIdle }),
page.ClickAsync("text=Click here to confirm your account"));
// Now we can login
await page.ClickAsync("text=Login");
await page.WaitForSelectorAsync("[name=\"Input.Email\"]");
await page.TypeAsync("[name=\"Input.Email\"]", userName);
await page.TypeAsync("[name=\"Input.Password\"]", password);
await page.FillAsync("[name=\"Input.Email\"]", userName);
await page.FillAsync("[name=\"Input.Password\"]", password);
await page.ClickAsync("#login-submit");
// Need to navigate to fetch page

Просмотреть файл

@ -205,7 +205,6 @@ public class BrowserManagerConfiguration
Env = configuration.GetValue<Dictionary<string, string>>(nameof(BrowserTypeLaunchOptions.Env)),
DownloadsPath = configuration.GetValue<string>(nameof(BrowserTypeLaunchOptions.DownloadsPath)),
ExecutablePath = configuration.GetValue<string>(nameof(BrowserTypeLaunchOptions.ExecutablePath)),
Devtools = configuration.GetValue<bool?>(nameof(BrowserTypeLaunchOptions.Devtools)),
Args = BindMultiValueMap(
configuration.GetSection(nameof(BrowserTypeLaunchOptions.Args)),
argsMap => argsMap.SelectMany(argNameValue => argNameValue.Value.Prepend(argNameValue.Key)).ToArray()),
@ -342,7 +341,6 @@ public class BrowserManagerConfiguration
Env = overrideOptions.Env != default ? overrideOptions.Env : defaultOptions.Env,
DownloadsPath = overrideOptions.DownloadsPath != default ? overrideOptions.DownloadsPath : defaultOptions.DownloadsPath,
ExecutablePath = overrideOptions.ExecutablePath != default ? overrideOptions.ExecutablePath : defaultOptions.ExecutablePath,
Devtools = overrideOptions.Devtools != default ? overrideOptions.Devtools : defaultOptions.Devtools,
Args = overrideOptions.Args != default ? overrideOptions.Args : defaultOptions.Args,
Headless = overrideOptions.Headless != default ? overrideOptions.Headless : defaultOptions.Headless,
Timeout = overrideOptions.Timeout != default ? overrideOptions.Timeout : defaultOptions.Timeout,