Bug 1851438 - [remote] Fix expectations and tests for Puppeteer v21.2.0 on mozilla CI r=webdriver-reviewers,whimboo

Depends on D188104

> TestExpectations.json
> Launcher specs Puppeteer Puppeteer.launch should launch Chrome properly with

Those tests can't work on Mozilla CI because we don't perform a regular install of puppeteer there.

> TestExpectations.json
> Launcher specs Puppeteer Puppeteer.launch should work with no default arguments

Same thing.

> TestExpectations.json
> [proxy.spec] request proxy should respect proxy bypass list

This test frequently fails on our CI because the responseCompleted event is received too late (ie after navigate has resolved). Filed https://bugzilla.mozilla.org/show_bug.cgi?id=1852891 about this.

> TestExpectations.json
> [network.spec] network Response.fromCache should work

I am not sure how this one passes on the puppeteer CI. Locally I never get a second request for the CSS file which we expect to get from the cache. Maybe we are completely missing the request in this case, but I don't know how this can pass on the puppeteer CI.

> TestExpectations.json
> [waittask.spec] waittask specs Frame.waitForFunction should work when resolved right before execution context disposal

The polling mechanism used in this test always fail for me locally and on CI. It might pass on puppeteer CI if it has slightly different timings?

> evaluation.spec.ts
> should not throw an error when evaluation does a navigation

Is ending too quickly on our CI, and the next test is trying to use the browsing context which is navigating, failing frequently.

> page.spec.ts
> Page.Events.PageError should fire

I frequently receive other errors for the same context during the test, which make the test fail. I'm proposing to wait for the specific error message here to avoid the issue.

> target.spec.ts
> should timeout waiting for a non-existent target

Depending on which test ran before, there could be a `server.EMPTY_PAGE` target available, and for some reason that lead the test to immediately fail with a TIMEOUT. Not quite sure why, but I think using an obviously not matching URL is safer?

Differential Revision: https://phabricator.services.mozilla.com/D188105

remote/test/puppeteer/test/TestExpectations.json

@@ -1211,6 +1211,12 @@
     "parameters": ["webDriverBiDi"],
     "expectations": ["PASS"]
   },
+  {
+    "testIdPattern": "[proxy.spec] request proxy should respect proxy bypass list",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["webDriverBiDi", "firefox"],
+    "expectations": ["PASS", "FAIL"]
+  },
   {
     "testIdPattern": "[queryhandler.spec] Query handler tests P selectors should work ARIA selectors",
     "platforms": ["darwin", "linux", "win32"],
@@ -2318,20 +2324,14 @@
   {
     "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch should launch Chrome properly with --no-startup-window and waitForInitialPage=false",
     "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["firefox", "webDriverBiDi"],
-    "expectations": ["FAIL"]
-  },
-  {
-    "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch should launch Chrome properly with --no-startup-window and waitForInitialPage=false",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["cdp", "firefox"],
+    "parameters": ["firefox"],
     "expectations": ["SKIP"]
   },
   {
     "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch should work with no default arguments",
     "platforms": ["darwin", "linux", "win32"],
     "parameters": ["firefox", "headless"],
-    "expectations": ["FAIL", "PASS"]
+    "expectations": ["SKIP"]
   },
   {
     "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch tmp profile should be cleaned up",
@@ -2945,6 +2945,12 @@
     "parameters": ["cdp", "firefox"],
     "expectations": ["FAIL"]
   },
+  {
+    "testIdPattern": "[network.spec] network Response.fromCache should work",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["webDriverBiDi", "firefox"],
+    "expectations": ["PASS", "FAIL"]
+  },
   {
     "testIdPattern": "[network.spec] network Response.fromServiceWorker Response.fromServiceWorker",
     "platforms": ["darwin", "linux", "win32"],
@@ -3857,6 +3863,12 @@
     "parameters": ["cdp", "firefox"],
     "expectations": ["SKIP"]
   },
+  {
+    "testIdPattern": "[waittask.spec] waittask specs Frame.waitForFunction should work when resolved right before execution context disposal",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["webDriverBiDi", "firefox"],
+    "expectations": ["PASS", "FAIL"]
+  },
   {
     "testIdPattern": "[waittask.spec] waittask specs Frame.waitForFunction should work with strict CSP policy",
     "platforms": ["darwin", "linux", "win32"],

remote/test/puppeteer/test/src/evaluation.spec.ts

@@ -423,11 +423,13 @@ describe('Evaluation specs', function () {
       const {page, server} = await getTestState();
 
       await page.goto(server.PREFIX + '/one-style.html');
+      const onRequest = server.waitForRequest('/empty.html');
       const result = await page.evaluate(() => {
         (window as any).location = '/empty.html';
         return [42];
       });
       expect(result).toEqual([42]);
+      await onRequest;
     });
     it('should transfer 100Mb of data from page to node.js', async function () {
       this.timeout(25_000);

remote/test/puppeteer/test/src/page.spec.ts

@@ -1364,7 +1364,9 @@ describe('Page', function () {
       const {page, server} = await getTestState();
 
       const [error] = await Promise.all([
-        waitEvent<Error>(page, 'pageerror'),
+        waitEvent<Error>(page, 'pageerror', err => {
+          return err.message.includes('Fancy');
+        }),
         page.goto(server.PREFIX + '/error.html'),
       ]);
       expect(error.message).toContain('Fancy');

remote/test/puppeteer/test/src/target.spec.ts

@@ -336,7 +336,7 @@ describe('Target', function () {
       await browser
         .waitForTarget(
           target => {
-            return target.url() === server.EMPTY_PAGE;
+            return target.url() === server.PREFIX + '/does-not-exist.html';
           },
           {
             timeout: 1,