test: fix NUnit Console.Error and friends to appear in test output (#3027)

.github/workflows/tests.yml

@@ -19,7 +19,7 @@ jobs:
       fail-fast: false
       matrix:
         browser: [chromium, firefox, webkit]
-        os: [windows-latest, ubuntu-latest, macos-13]
+        os: [windows-latest, ubuntu-latest, macos-latest]
     steps:
       - uses: actions/checkout@v3
       - name: Setup .NET Core

src/Playwright.Tests.TestServer/Playwright.Tests.TestServer.csproj

@@ -16,6 +16,9 @@
   <ItemGroup>
     <Folder Include="assets\" />
   </ItemGroup>
+  <ItemGroup>
+    <PackageReference Include="NUnit" Version="4.2.2" />
+  </ItemGroup>
   <Target Name="CheckAssetsFolderExists" BeforeTargets="Build">
     <Error Text="assets folder prerequisites are missing. Ensure you've ran `.\build.sh --init` from the root of the solution." Condition="!Exists('$(MSBuildProjectDirectory)\assets\empty.html')" />
   </Target>

src/Playwright.Tests.TestServer/SimpleServer.cs

@@ -30,6 +30,7 @@ using System.IO;
 using System.IO.Compression;
 using System.Net;
 using System.Net.WebSockets;
+using System.Reflection;
 using System.Security.Cryptography.X509Certificates;
 using System.Text;
 using System.Text.RegularExpressions;
@@ -41,6 +42,8 @@ using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Http.Features;
 using Microsoft.AspNetCore.StaticFiles;
 using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using NUnit.Framework.Internal;
 
 namespace Microsoft.Playwright.Tests.TestServer;
 
@@ -80,6 +83,7 @@ public class SimpleServer
 
         EmptyPage = $"{Prefix}/empty.html";
 
+        var currentExecutionContext = TestExecutionContext.CurrentContext;
         _requestWaits = new ConcurrentDictionary<string, Action<HttpContext>>();
         _waitForWebSocketConnectionRequestsWaits = [];
         _routes = new ConcurrentDictionary<string, Func<HttpContext, Task>>();
@@ -89,11 +93,21 @@ public class SimpleServer
         _contentRoot = contentRoot;
 
         _webHost = new WebHostBuilder()
+            .ConfigureLogging(logging =>
+            {
+                // Allow seeing exceptions in the console output.
+                logging.AddConsole();
+                logging.SetMinimumLevel(LogLevel.Error);
+            })
             .Configure((app) => app
                 .UseWebSockets()
-                .UseDeveloperExceptionPage()
                 .Use(middleware: async (HttpContext context, Func<Task> next) =>
                 {
+                    {
+                        // This hack allows us to have Console.WriteLine etc. appear in the test output.
+                        var currentContext = typeof(TestExecutionContext).GetField("AsyncLocalCurrentContext", BindingFlags.NonPublic | BindingFlags.Static).GetValue(null) as AsyncLocal<TestExecutionContext>;
+                        currentContext.Value = currentExecutionContext;
+                    }
                     if (context.Request.Path == "/ws")
                     {
                         if (context.WebSockets.IsWebSocketRequest)
@@ -236,7 +250,7 @@ public class SimpleServer
 
     public void SetCSP(string path, string csp) => _csp.Add(path, csp);
 
-    public Task StartAsync() => _webHost.StartAsync();
+    public Task StartAsync(CancellationToken cancellationToken) => _webHost.StartAsync(cancellationToken);
 
     public Task StopAsync()
     {

src/Playwright.Tests/BaseTests/HttpService.cs

@@ -42,7 +42,7 @@ public class HttpService : IWorkerService
                 Server = SimpleServer.Create(8907 + workerIndex * 2, assetDir),
                 HttpsServer = SimpleServer.CreateHttps(8907 + workerIndex * 2 + 1, assetDir)
             };
-            await Task.WhenAll(http.Server.StartAsync(), http.HttpsServer.StartAsync());
+            await Task.WhenAll(http.Server.StartAsync(TestContext.CurrentContext.CancellationToken), http.HttpsServer.StartAsync(TestContext.CurrentContext.CancellationToken));
             return http;
         });
     }

src/Playwright.Tests/BaseTests/PlaywrightAssert.cs

@@ -23,7 +23,6 @@
  */
 
 using System.Text.Json;
-using NUnit.Framework.Constraints;
 using SixLabors.ImageSharp;
 using SixLabors.ImageSharp.PixelFormats;
 

src/Playwright.Tests/BrowserContextViewportMobileTests.cs

@@ -70,8 +70,7 @@ public class BrowserContextViewportMobileTests : BrowserTestEx
     {
         await using var context = await Browser.NewContextAsync(Playwright.Devices["iPhone 6"]);
         var page = await context.NewPageAsync();
-        await page.GotoAsync(Server.Prefix + "/detect-touch.html");
-        Assert.AreEqual("YES", await page.EvaluateAsync<string>("document.body.textContent.trim()"));
+        Assert.AreEqual(true, await page.EvaluateAsync<bool>("'ontouchstart' in window || !!window.TouchEvent"));
     }
 
     [PlaywrightTest("browsercontext-viewport-mobile.spec.ts", "should detect touch when applying viewport with touches")]

src/Playwright.Tests/BrowserContextViewportTests.cs

@@ -86,8 +86,7 @@ public class BrowserContextViewportTests : PageTestEx
     {
         await Page.GotoAsync(Server.Prefix + "/mobile.html");
         Assert.False(await Page.EvaluateAsync<bool>("'ontouchstart' in window"));
-        await Page.GotoAsync(Server.Prefix + "/detect-touch.html");
-        Assert.AreEqual("NO", await Page.EvaluateAsync<string>("document.body.textContent.trim()"));
+        Assert.AreEqual(false, await Page.EvaluateAsync<bool>("'ontouchstart' in window"));
     }
 
     [PlaywrightTest("browsercontext-viewport.spec.ts", "should support touch with null viewport")]

src/Playwright.Tests/BrowserTypeConnectTests.cs

@@ -453,7 +453,7 @@ public class BrowserTypeConnectTests : PlaywrightTestEx
     }
 
     [PlaywrightTest("browsertype-connect.spec.ts", "should upload large file")]
-    [Timeout(TestConstants.SlowTestTimeout)]
+    [CancelAfter(TestConstants.SlowTestTimeout)]
     public async Task ShouldUploadLargeFile()
     {
         var browser = await BrowserType.ConnectAsync(_remoteServer.WSEndpoint);

src/Playwright.Tests/GlobalUsings.cs

@@ -0,0 +1,29 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and / or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// We should eventually migrate to the constrained assertions:
+// https://docs.nunit.org/articles/nunit/release-notes/Nunit4.0-MigrationGuide.html#use-global-using-aliases
+global using Assert = NUnit.Framework.Legacy.ClassicAssert;
+global using CollectionAssert = NUnit.Framework.Legacy.CollectionAssert;
+global using StringAssert = NUnit.Framework.Legacy.StringAssert;

src/Playwright.Tests/PageGotoTests.cs

@@ -279,7 +279,7 @@ public class PageGotoTests : PageTestEx
         }
         else if (TestConstants.IsWebKit && TestConstants.IsWindows)
         {
-            StringAssert.Contains("Couldn't connect to server", exception.Message);
+            StringAssert.Contains("Could not connect to server", exception.Message);
         }
         else if (TestConstants.IsWebKit)
         {

src/Playwright.Tests/PageSetInputFilesTests.cs

@@ -353,7 +353,7 @@ public class PageSetInputFilesTests : PageTestEx
     }
 
     [PlaywrightTest("page-set-input-files.spec.ts", "should upload large file")]
-    [Timeout(TestConstants.SlowTestTimeout)]
+    [CancelAfter(TestConstants.SlowTestTimeout)]
     public async Task ShouldUploadLargeFile()
     {
         await Page.GotoAsync(Server.Prefix + "/input/fileupload.html");

src/Playwright.Tests/PageWaitForNavigationTests.cs

@@ -307,7 +307,7 @@ public class PageWaitForNavigationTests : PageTestEx
     }
 
     [PlaywrightTest]
-    [Timeout(45_000)]
+    [CancelAfter(45_000)]
     public async Task ShouldHaveADefaultTimeout()
     {
         await Page.GotoAsync(Server.Prefix + "/frames/one-frame.html");
@@ -315,7 +315,7 @@ public class PageWaitForNavigationTests : PageTestEx
     }
 
     [PlaywrightTest]
-    [Timeout(5_000)]
+    [CancelAfter(5_000)]
     public async Task ShouldTakeTimeoutIntoAccount()
     {
         await Page.GotoAsync(Server.Prefix + "/frames/one-frame.html");

src/Playwright.Tests/Playwright.Tests.csproj

@@ -21,9 +21,9 @@
           <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
           <PrivateAssets>all</PrivateAssets>
         </PackageReference>
-        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.8.0" />
-        <PackageReference Include="NUnit" Version="3.14.0" />
-        <PackageReference Include="NUnit3TestAdapter" Version="4.5.0" />
+        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
+        <PackageReference Include="NUnit" Version="4.2.2" />
+        <PackageReference Include="NUnit3TestAdapter" Version="4.6.0" />
         <PackageReference Include="SixLabors.ImageSharp" Version="2.1.9" />
     </ItemGroup>
     <ItemGroup>

src/Playwright.Tests/ScreencastTests.cs

@@ -191,7 +191,7 @@ public class ScreencastTests : BrowserTestEx
 
 
     [PlaywrightTest("screencast.spec.ts", "video.path()/saveAs() does not hang immediately after launchPersistentContext and context.close()")]
-    [Timeout(30_000)]
+    [CancelAfter(30_000)]
     public async Task VideoPathSaveAsDoesNotHangImmediatelyAfterLaunchPersistentContextAndContextClose()
     {
         using var userDirectory = new TempDirectory();