Bug 1627687 - Make console complicated reload test more reliable. r=jdescottes,perftest-reviewers,sparky.

This patch modifies the reloadConsoleAndLog helper so it can
consumes an array of expected messages, and not only a number
of expected messages.
This should prevent the performance variations caused by new
warning messages being added.

Differential Revision: https://phabricator.services.mozilla.com/D69816

--HG--
extra : moz-landing-system : lando

testing/talos/talos/tests/devtools/addon/content/tests/webconsole/complicated.js

@@ -14,14 +14,44 @@ const {
 } = require("../head");
 const { reloadConsoleAndLog } = require("./webconsole-helpers");
 
-const EXPECTED_MESSAGES = isFissionEnabled() ? 3 : 7;
+const EXPECTED_MESSAGES = [
+  {
+    text: `This page uses the non standard property “zoom”`,
+    count: isFissionEnabled() ? 2 : 4,
+    visibleWhenFissionEnabled: true,
+  },
+  {
+    text: `Layout was forced before the page was fully loaded.`,
+    visibleWhenFissionEnabled: true,
+  },
+  {
+    text: `Some cookies are misusing the “sameSite“ attribute, so it won’t work as expected`,
+    visibleWhenFissionEnabled: true,
+  },
+  {
+    text: `InvalidStateError: XMLHttpRequest state must be OPENED.`,
+    visibleWhenFissionEnabled: true,
+  },
+  {
+    text: `SyntaxError: missing ) after argument list`,
+    count: 2,
+    visibleWhenFissionEnabled: false,
+  },
+  {
+    text: `ReferenceError: Bootloaddisableder is not defined`,
+    count: 4,
+    visibleWhenFissionEnabled: false,
+  },
+].filter(
+  ({ visibleWhenFissionEnabled }) =>
+    !isFissionEnabled() || visibleWhenFissionEnabled
+);
 
 module.exports = async function() {
   await testSetup(COMPLICATED_URL);
 
   let toolbox = await openToolboxAndLog("complicated.webconsole", "webconsole");
   await reloadConsoleAndLog("complicated", toolbox, EXPECTED_MESSAGES);
-
   await closeToolboxAndLog("complicated.webconsole", toolbox);
 
   await testTeardown();

testing/talos/talos/tests/devtools/addon/content/tests/webconsole/webconsole-helpers.js

@@ -6,13 +6,66 @@
 
 const { reloadPageAndLog } = require("../head");
 
+/**
+ * @param {String} label: The name of the test.
+ * @param {Toolbox} toolbox: The DevTools toolbox.
+ * @param {Number|Array} expectedMessages: This can be, either the number of messages that
+ *        need to be displayed in the console, or an array of objects representing the
+ *        messages that need to be in the output. The objects must have the following shape:
+ *          - {String} text: A string that should be in the message.
+ *          - {Number} count: If > 1, indicate how many messages with this text should be
+ *                            in the output.
+ */
 exports.reloadConsoleAndLog = async function(label, toolbox, expectedMessages) {
   const onReload = async function() {
     const { hud } = toolbox.getPanel("webconsole");
 
+    const expected =
+      typeof expectedMessages === "number"
+        ? [{ text: "", count: expectedMessages }]
+        : expectedMessages;
+
     await waitForConsoleOutputChildListChange(hud, consoleOutputEl => {
-      const messageCount = consoleOutputEl.querySelectorAll(".message").length;
-      return messageCount >= expectedMessages;
+      dump("[TEST_LOG] Console output changed - checking content:\n");
+      const messages = Array.from(consoleOutputEl.querySelectorAll(".message"));
+
+      const missing = new Map(expected.map(e => [e.text, e.count || 1]));
+
+      const foundAllMessages = expected.every(({ text, count = 1 }) => {
+        let found = 0;
+        for (const message of messages) {
+          const messageText = message.querySelector(".message-body").innerText;
+          if (messageText.includes(text)) {
+            const repeat = message
+              .querySelector(".message-repeats")
+              ?.innerText?.trim();
+            found = found + (repeat ? parseInt(repeat) : 1);
+          }
+        }
+        const allFound = found >= count;
+
+        if (allFound) {
+          missing.delete(text);
+        } else {
+          missing.set(text, count - found);
+        }
+
+        return allFound;
+      });
+
+      if (!foundAllMessages) {
+        dump(
+          `[TEST_LOG] Still waiting for the following messages: \n${Array.from(
+            missing.entries()
+          )
+            .map(([text, count]) => `${text || "<any text>"} (✕${count})`)
+            .join("\n")}\n`
+        );
+      } else {
+        dump(`[TEST_LOG] All expected messages where found\n`);
+      }
+      dump("---\n");
+      return foundAllMessages;
     });
   };
   await reloadPageAndLog(label + ".webconsole", toolbox, onReload);