Bug 1880571 - mach manifest skip-fails: initial WPT support r=jmaher

Differential Revision: https://phabricator.services.mozilla.com/D203038

testing/mach_commands.py

@@ -1255,7 +1255,11 @@ def manifest(_command_context):
 )
 @CommandArgument("try_url", nargs=1, help="Treeherder URL for try (please use quotes)")
 @CommandArgument(
-    "-b", "--bugzilla", default=None, dest="bugzilla", help="Bugzilla instance"
+    "-b",
+    "--bugzilla",
+    default=None,
+    dest="bugzilla",
+    help="Bugzilla instance (or disable)",
 )
 @CommandArgument(
     "-m", "--meta-bug-id", default=None, dest="meta_bug_id", help="Meta Bug id"

testing/test/data/wayland-failures-1.json

@@ -2,19 +2,22 @@
   "browser/base/content/test/performance/browser.toml": {
     "label": {
       "test-windows11-64-2009-qr/opt-mochitest-browser-chrome-3": {
+        "duration_median": 63094,
+        "duration_total": 63094,
         "durations": {
           "dwOJ8M9ERSmk6oI2KXg6hg": 63094
         },
-        "median_duration": 63094,
         "opt": false,
         "path": {
           "browser/base/content/test/performance/browser_startup.js": {
             "classification": "unknown",
-            "failed_runs": 1,
             "runs": {
-              "dwOJ8M9ERSmk6oI2KXg6hg": false
+              "dwOJ8M9ERSmk6oI2KXg6hg": {
+                "result": false
+              }
             },
-            "total_runs": 1
+            "runs_failed": 1,
+            "runs_total": 1
           }
         },
         "sum_by_label": {
@@ -25,9 +28,9 @@
           "secondary": 0,
           "success": 0,
           "unknown": 1
-        },
-        "total_duration": 63094
+        }
       }
-    }
+    },
+    "wpt": false
   }
 }

testing/test/data/wayland-failures-2.json

@@ -2,33 +2,46 @@
   "browser/components/sessionstore/test/browser.toml": {
     "label": {
       "test-linux1804-64-qr/opt-mochitest-browser-chrome-spi-nw-5": {
+        "duration_median": 1778154,
+        "duration_total": 5334462,
         "durations": {
           "X7r1q2xWSu-2bRAofEfeBw": 1778154,
           "Y7r1q2xWSu-2bRAofEfeBw": 1778154,
           "Z7r1q2xWSu-2bRAofEfeBw": 1778154
         },
-        "median_duration": 1778154,
         "opt": false,
         "path": {
           "browser/components/sessionstore/test/browser_closed_tabs_windows.js": {
             "classification": "disable_recommended",
-            "failed_runs": 3,
             "runs": {
-              "X7r1q2xWSu-2bRAofEfeBw": false,
-              "Y7r1q2xWSu-2bRAofEfeBw": false,
-              "Z7r1q2xWSu-2bRAofEfeBw": false
+              "X7r1q2xWSu-2bRAofEfeBw": {
+                "result": false
+              },
+              "Y7r1q2xWSu-2bRAofEfeBw": {
+                "result": false
+              },
+              "Z7r1q2xWSu-2bRAofEfeBw": {
+                "result": false
+              }
             },
-            "total_runs": 3
+            "runs_failed": 3,
+            "runs_total": 3
           },
           "browser/components/sessionstore/test/browser_firefoxView_selected_restore.js": {
             "classification": "secondary",
-            "failed_runs": 3,
             "runs": {
-              "X7r1q2xWSu-2bRAofEfeBw": false,
-              "Y7r1q2xWSu-2bRAofEfeBw": false,
-              "Z7r1q2xWSu-2bRAofEfeBw": false
+              "X7r1q2xWSu-2bRAofEfeBw": {
+                "result": false
+              },
+              "Y7r1q2xWSu-2bRAofEfeBw": {
+                "result": false
+              },
+              "Z7r1q2xWSu-2bRAofEfeBw": {
+                "result": false
+              }
             },
-            "total_runs": 3
+            "runs_failed": 3,
+            "runs_total": 3
           }
         },
         "sum_by_label": {
@@ -39,9 +52,9 @@
           "secondary": 1,
           "success": 0,
           "unknown": 0
-        },
-        "total_duration": 5334462
+        }
       }
-    }
+    },
+    "wpt": false
   }
 }

testing/test/data/wayland-failures-3.json

@@ -2,23 +2,30 @@
   "browser/components/urlbar/tests/browser-updateResults/browser.toml": {
     "label": {
       "test-linux1804-64-qr/opt-mochitest-browser-chrome-spi-nw-7": {
+        "duration_median": 82817,
+        "duration_total": 243314,
         "durations": {
           "UOZUIVAaTZKmRwArq5WkDw": 55841,
           "WVczuxkuSRKZg_jMiGyQsA": 82817,
           "b7_ahjGtQ_-ZMNBG_hUZUw": 104656
         },
-        "median_duration": 82817,
         "opt": false,
         "path": {
           "browser/components/urlbar/tests/browser-updateResults/browser_suggestedIndex_10_url_10_search.js": {
             "classification": "intermittent",
-            "failed_runs": 1,
             "runs": {
-              "UOZUIVAaTZKmRwArq5WkDw": false,
-              "WVczuxkuSRKZg_jMiGyQsA": true,
-              "b7_ahjGtQ_-ZMNBG_hUZUw": true
+              "UOZUIVAaTZKmRwArq5WkDw": {
+                "result": false
+              },
+              "WVczuxkuSRKZg_jMiGyQsA": {
+                "result": true
+              },
+              "b7_ahjGtQ_-ZMNBG_hUZUw": {
+                "result": true
+              }
             },
-            "total_runs": 3
+            "runs_failed": 1,
+            "runs_total": 3
           }
         },
         "sum_by_label": {
@@ -29,9 +36,9 @@
           "secondary": 0,
           "success": 0,
           "unknown": 0
-        },
-        "total_duration": 243314
+        }
       }
-    }
+    },
+    "wpt": false
   }
 }

testing/test/data/wayland-failures-4.json

@@ -2,23 +2,30 @@
   "toolkit/components/pdfjs/test/browser.toml": {
     "label": {
       "test-linux2204-64-wayland/debug-mochitest-browser-chrome-swr-1": {
+        "duration_median": 390469,
+        "duration_total": 1171407,
         "durations": {
           "EDql3NKPR3W6OEU3mLeKbg": 390469,
           "FDql3NKPR3W6OEU3mLeKbg": 390469,
           "bxMVPbPMTru_bfAivc1sPA": 390469
         },
-        "median_duration": 390469,
         "opt": false,
         "path": {
           "DEFAULT": {
             "classification": "disable_manifest",
-            "failed_runs": 3,
             "runs": {
-              "EDql3NKPR3W6OEU3mLeKbg": false,
-              "FDql3NKPR3W6OEU3mLeKbg": false,
-              "bxMVPbPMTru_bfAivc1sPA": false
+              "EDql3NKPR3W6OEU3mLeKbg": {
+                "result": false
+              },
+              "FDql3NKPR3W6OEU3mLeKbg": {
+                "result": false
+              },
+              "bxMVPbPMTru_bfAivc1sPA": {
+                "result": false
+              }
             },
-            "total_runs": 3
+            "runs_failed": 3,
+            "runs_total": 3
           }
         },
         "sum_by_label": {
@@ -29,9 +36,9 @@
           "secondary": 0,
           "success": 0,
           "unknown": 0
-        },
-        "total_duration": 1171407
+        }
       }
-    }
+    },
+    "wpt": false
   }
 }

testing/test/data/wayland-failures-5.json

@@ -2,54 +2,59 @@
   "browser/components/places/tests/browser/browser.toml": {
     "label": {
       "test-linux2204-64-wayland/opt-mochitest-browser-chrome-swr-3": {
+        "duration_median": 1407021,
+        "duration_total": 4249131,
         "durations": {
           "Bgc6We1sSjakIo3V9crldw": 1521792,
           "EH0WPmGtQsGzu5K7OxFOyg": 1407021,
           "bxDhZzsxQw65gn-veX1PHg": 1320318
         },
-        "median_duration": 1407021,
-        "opt": true,
+        "opt": false,
         "path": {
-          "DEFAULT": {
-            "classification": "disable_too_long",
-            "failed_runs": 1,
-            "runs": {
-              "bxDhZzsxQw65gn-veX1PHg": false
-            },
-            "total_runs": 1
-          },
           "browser/components/places/tests/browser/browser_bookmarkProperties_addKeywordForThisSearch.js": {
             "classification": "disable_recommended",
-            "failed_runs": 3,
             "runs": {
-              "Bgc6We1sSjakIo3V9crldw": false,
-              "EH0WPmGtQsGzu5K7OxFOyg": false,
-              "bxDhZzsxQw65gn-veX1PHg": false
+              "Bgc6We1sSjakIo3V9crldw": {
+                "result": false
+              },
+              "EH0WPmGtQsGzu5K7OxFOyg": {
+                "result": false
+              },
+              "bxDhZzsxQw65gn-veX1PHg": {
+                "result": false
+              }
             },
-            "total_runs": 3
+            "runs_failed": 3,
+            "runs_total": 3
           },
           "browser/components/places/tests/browser/browser_bookmark_popup.js": {
             "classification": "secondary",
-            "failed_runs": 3,
             "runs": {
-              "Bgc6We1sSjakIo3V9crldw": false,
-              "EH0WPmGtQsGzu5K7OxFOyg": false,
-              "bxDhZzsxQw65gn-veX1PHg": false
+              "Bgc6We1sSjakIo3V9crldw": {
+                "result": false
+              },
+              "EH0WPmGtQsGzu5K7OxFOyg": {
+                "result": false
+              },
+              "bxDhZzsxQw65gn-veX1PHg": {
+                "result": false
+              }
             },
-            "total_runs": 3
+            "runs_failed": 3,
+            "runs_total": 3
           }
         },
         "sum_by_label": {
           "disable_manifest": 0,
           "disable_recommended": 1,
-          "disable_too_long": 1,
+          "disable_too_long": 0,
           "intermittent": 0,
           "secondary": 1,
           "success": 0,
           "unknown": 0
-        },
-        "total_duration": 4249131
+        }
       }
-    }
+    },
+    "wpt": false
   }
 }

testing/test/data/wpt-failures-1.json

@@ -0,0 +1,48 @@
+{
+  "testing/web-platform/meta/css/mediaqueries": {
+    "label": {
+      "test-linux1804-32-shippable-qr/opt-web-platform-tests-10": {
+        "duration_median": 7359.0,
+        "duration_total": 28063,
+        "durations": {
+          "AKYqxtoWStigj_5yHVqAeg": 7368,
+          "H_sxPcuASXGy2N6MOwvMdg": 7350,
+          "W5uCyxXeQIa2dwJOZtLYgg": 7724,
+          "fyZfG_PJSse7jeiDteDJpQ": 5621
+        },
+        "opt": false,
+        "path": {
+          "testing/web-platform/tests/css/mediaqueries/mq-non-matching-lazy-load.tentative.html": {
+            "classification": "disable_recommended",
+            "runs": {
+              "AKYqxtoWStigj_5yHVqAeg": {
+                "result": true
+              },
+              "H_sxPcuASXGy2N6MOwvMdg": {
+                "result": false
+              },
+              "W5uCyxXeQIa2dwJOZtLYgg": {
+                "result": false
+              },
+              "fyZfG_PJSse7jeiDteDJpQ": {
+                "result": true
+              }
+            },
+            "runs_failed": 2,
+            "runs_total": 4
+          }
+        },
+        "sum_by_label": {
+          "disable_manifest": 0,
+          "disable_recommended": 1,
+          "disable_too_long": 0,
+          "intermittent": 0,
+          "secondary": 0,
+          "success": 0,
+          "unknown": 0
+        }
+      }
+    },
+    "wpt": true
+  }
+}

testing/test/data/wpt-tasks-1.json

@@ -0,0 +1,164 @@
+[
+  {
+    "duration": 1988052,
+    "errors": null,
+    "extra": {
+      "arch": "32",
+      "bits": "32",
+      "build_types": ["opt", "shippable"],
+      "debug": false,
+      "display": null,
+      "opt": true,
+      "os": "linux",
+      "os_version": "18.04",
+      "runtimes": []
+    },
+    "failure_types": {
+      "/css/mediaqueries": [
+        [
+          "/css/mediaqueries/mq-non-matching-lazy-load.tentative.html",
+          "generic"
+        ]
+      ]
+    },
+    "id": "W5uCyxXeQIa2dwJOZtLYgg",
+    "label": "test-linux1804-32-shippable-qr/opt-web-platform-tests-10",
+    "result": "failed",
+    "results": [
+      {
+        "duration": 7724,
+        "group": "testing/web-platform/tests/css/mediaqueries",
+        "ok": true
+      }
+    ],
+    "state": "completed",
+    "tags": {
+      "createdForUser": "ci@mozilla.com",
+      "kind": "test",
+      "label": "test-linux1804-32-shippable-qr/opt-web-platform-tests-10",
+      "os": "linux",
+      "retrigger": "true",
+      "tests_grouped": "1",
+      "worker-implementation": "docker-worker"
+    },
+    "tier": 2
+  },
+  {
+    "duration": 1918629,
+    "errors": null,
+    "extra": {
+      "arch": "32",
+      "bits": "32",
+      "build_types": ["opt", "shippable"],
+      "debug": false,
+      "display": null,
+      "opt": true,
+      "os": "linux",
+      "os_version": "18.04",
+      "runtimes": []
+    },
+    "failure_types": {},
+    "id": "AKYqxtoWStigj_5yHVqAeg",
+    "label": "test-linux1804-32-shippable-qr/opt-web-platform-tests-10",
+    "result": "failed",
+    "results": [
+      {
+        "duration": 7368,
+        "group": "testing/web-platform/tests/css/mediaqueries",
+        "ok": true
+      }
+    ],
+    "state": "completed",
+    "tags": {
+      "createdForUser": "ci@mozilla.com",
+      "kind": "test",
+      "label": "test-linux1804-32-shippable-qr/opt-web-platform-tests-10",
+      "os": "linux",
+      "retrigger": "true",
+      "tests_grouped": "1",
+      "worker-implementation": "docker-worker"
+    },
+    "tier": 2
+  },
+  {
+    "duration": 1875924,
+    "errors": null,
+    "extra": {
+      "arch": "32",
+      "bits": "32",
+      "build_types": ["opt", "shippable"],
+      "debug": false,
+      "display": null,
+      "opt": true,
+      "os": "linux",
+      "os_version": "18.04",
+      "runtimes": []
+    },
+    "failure_types": {
+      "/css/mediaqueries": [
+        [
+          "/css/mediaqueries/mq-non-matching-lazy-load.tentative.html",
+          "generic"
+        ]
+      ]
+    },
+    "id": "H_sxPcuASXGy2N6MOwvMdg",
+    "label": "test-linux1804-32-shippable-qr/opt-web-platform-tests-10",
+    "result": "failed",
+    "results": [
+      {
+        "duration": 7350,
+        "group": "testing/web-platform/tests/css/mediaqueries",
+        "ok": true
+      }
+    ],
+    "state": "completed",
+    "tags": {
+      "createdForUser": "ci@mozilla.com",
+      "kind": "test",
+      "label": "test-linux1804-32-shippable-qr/opt-web-platform-tests-10",
+      "os": "linux",
+      "retrigger": "true",
+      "tests_grouped": "1",
+      "worker-implementation": "docker-worker"
+    },
+    "tier": 2
+  },
+  {
+    "duration": 1560033,
+    "errors": null,
+    "extra": {
+      "arch": "32",
+      "bits": "32",
+      "build_types": ["opt", "shippable"],
+      "debug": false,
+      "display": null,
+      "opt": true,
+      "os": "linux",
+      "os_version": "18.04",
+      "runtimes": []
+    },
+    "failure_types": {},
+    "id": "fyZfG_PJSse7jeiDteDJpQ",
+    "label": "test-linux1804-32-shippable-qr/opt-web-platform-tests-10",
+    "result": "failed",
+    "results": [
+      {
+        "duration": 5621,
+        "group": "testing/web-platform/tests/css/mediaqueries",
+        "ok": true
+      }
+    ],
+    "state": "completed",
+    "tags": {
+      "createdForUser": "ci@mozilla.com",
+      "kind": "test",
+      "label": "test-linux1804-32-shippable-qr/opt-web-platform-tests-10",
+      "os": "linux",
+      "retrigger": "true",
+      "tests_grouped": "1",
+      "worker-implementation": "docker-worker"
+    },
+    "tier": 2
+  }
+]

testing/test/test_skipfails.py

@@ -8,13 +8,7 @@ from pathlib import Path
 
 import pytest
 from mozunit import main
-from skipfails import (
-    MOCK_BUG_DEFAULTS,
-    MOCK_TASK_DEFAULTS,
-    MOCK_TASK_INITS,
-    Mock,
-    Skipfails,
-)
+from skipfails import Skipfails
 
 DATA_PATH = Path(__file__).with_name("data")
 
@@ -63,11 +57,9 @@ def get_failures(tasks_name, exp_f_name, task_details=None):
     if task_details is not None:  # preload task details cache, if needed
         for task_id in task_details:
             sf.tasks[task_id] = task_details[task_id]
-    tasks_fp = DATA_PATH.joinpath(tasks_name).open("r", encoding="utf-8")
-    tasks = json.load(tasks_fp)
-    tasks = [Mock(task, MOCK_TASK_DEFAULTS, MOCK_TASK_INITS) for task in tasks]
-    exp_f_fp = DATA_PATH.joinpath(exp_f_name).open("r", encoding="utf-8")
-    expected_failures = exp_f_fp.read().strip()
+    tasks = sf.read_tasks(DATA_PATH.joinpath(tasks_name))
+    exp_f = sf.read_failures(DATA_PATH.joinpath(exp_f_name))
+    expected_failures = json.dumps(exp_f, indent=2, sort_keys=True).strip()
     failures = sf.get_failures(tasks)
     actual_failures = json.dumps(failures, indent=2, sort_keys=True).strip()
     assert actual_failures == expected_failures
@@ -132,6 +124,31 @@ def test_get_failures_5():
     get_failures("wayland-tasks-5.json", "wayland-failures-5.json", task_details)
 
 
+def test_get_failures_6():
+    """Test get_failures 6"""
+
+    task_details = {
+        "AKYqxtoWStigj_5yHVqAeg": {
+            "expires": "2024-03-19T03:29:11.050Z",
+            "extra": {
+                "suite": "web-platform-tests",
+                "test-setting": {
+                    "build": {
+                        "type": "opt",
+                        "shippable": True,
+                    },
+                    "platform": {
+                        "arch": "32",
+                        "os": {"name": "linux", "version": "1804"},
+                    },
+                    "runtime": {},
+                },
+            },
+        }
+    }
+    get_failures("wpt-tasks-1.json", "wpt-failures-1.json", task_details)
+
+
 def test_get_bug_by_id():
     """Test get_bug_by_id"""
 
@@ -139,12 +156,10 @@ def test_get_bug_by_id():
     id = 1682371
     # preload bug cache
     bugs_filename = f"bug-{id}.json"
-    bugs_fp = DATA_PATH.joinpath(bugs_filename).open("r", encoding="utf-8")
-    bugs_data = json.load(bugs_fp)
-    preload_bugs = [Mock(b, MOCK_BUG_DEFAULTS) for b in bugs_data]
-    sf.bugs = preload_bugs
+    sf.bugs = sf.read_bugs(DATA_PATH.joinpath(bugs_filename))
     # function under test
     bug = sf.get_bug_by_id(id)
+    assert bug is not None
     assert bug.id == id
     assert bug.product == "Testing"
     assert bug.component == "General"
@@ -161,10 +176,7 @@ def test_get_bugs_by_summary():
     id = 1682371
     # preload bug cache
     bugs_filename = f"bug-{id}.json"
-    bugs_fp = DATA_PATH.joinpath(bugs_filename).open("r", encoding="utf-8")
-    bugs_data = json.load(bugs_fp)
-    preload_bugs = [Mock(b, MOCK_BUG_DEFAULTS) for b in bugs_data]
-    sf.bugs = preload_bugs
+    sf.bugs = sf.read_bugs(DATA_PATH.joinpath(bugs_filename))
     # function under test
     summary = "create tool to quickly parse and identify all failures from a try push and ideally annotate manifests"
     bugs = sf.get_bugs_by_summary(summary)
@@ -191,6 +203,7 @@ def test_get_variants():
 def test_task_to_skip_if():
     """Test task_to_skip_if"""
 
+    sf = Skipfails()
     # preload task cache
     task_id = "UP-t3xrGSDWvUNjFGIt_aQ"
     task_details = {
@@ -207,11 +220,231 @@ def test_task_to_skip_if():
             },
         },
     }
-    sf = Skipfails()
     sf.tasks[task_id] = task_details
     # function under test
     skip_if = sf.task_to_skip_if(task_id)
-    assert skip_if == "os == 'win' && os_version == '11' && bits == '32' && debug"
+    assert skip_if == "win11_2009 && processor == 'x86' && debug"
+    task_id = "I3iXyGDATDSDyzGh4YfNJw"
+    task_details = {
+        "expires": "2024-01-09T16:05:56.825Z",
+        "extra": {
+            "suite": "web-platform-tests-crashtest",
+            "test-setting": {
+                "build": {"type": "debug"},
+                "platform": {
+                    "arch": "64",
+                    "os": {"name": "macosx", "version": "1015"},
+                },
+                "runtime": {"webrender-sw": True},
+            },
+        },
+    }
+    sf.tasks[task_id] = task_details
+    # function under test
+    skip_if = sf.task_to_skip_if(task_id)
+    assert (
+        skip_if
+        == "os == 'mac' && os_version == '10.15' && processor == 'x86_64' && debug && swgl"
+    )
+    task_id = "bAkMaQIVQp6oeEIW6fzBDw"
+    task_details = {
+        "expires": "2024-01-09T16:05:56.825Z",
+        "extra": {
+            "suite": "mochitest-media",
+            "test-setting": {
+                "build": {"type": "debug"},
+                "platform": {
+                    "arch": "aarch64",
+                    "os": {"name": "macosx", "version": "1100"},
+                },
+                "runtime": {"webrender-sw": True},
+            },
+        },
+    }
+    sf.tasks[task_id] = task_details
+    # function under test
+    skip_if = sf.task_to_skip_if(task_id)
+    assert (
+        skip_if
+        == "os == 'mac' && os_version == '11.00' && processor == 'aarch64' && debug && swgl"
+    )
+
+
+def test_task_to_skip_if_wpt():
+    """Test task_to_skip_if_wpt"""
+
+    # preload task cache
+    task_id = "AKYqxtoWStigj_5yHVqAeg"
+    task_details = {
+        "expires": "2024-03-19T03:29:11.050Z",
+        "extra": {
+            "suite": "web-platform-tests",
+            "test-setting": {
+                "build": {
+                    "type": "opt",
+                    "shippable": True,
+                },
+                "platform": {
+                    "arch": "32",
+                    "os": {"name": "linux", "version": "1804"},
+                },
+                "runtime": {},
+            },
+        },
+    }
+    sf = Skipfails()
+    sf.tasks[task_id] = task_details
+    # function under test
+    skip_if = sf.task_to_skip_if(task_id, True)
+    assert (
+        skip_if
+        == 'os == "linux" and os_version == "18.04" and processor == "x86" and not debug'
+    )
+
+
+def test_wpt_add_skip_if():
+    """Test wpt_add_skip_if"""
+
+    sf = Skipfails()
+    manifest_before1 = ""
+    anyjs = {}
+    filename = "myfile.html"
+    anyjs[filename] = False
+    skip_if = 'os == "linux" and processor == "x86" and not debug'
+    bug_reference = "Bug 123"
+    disabled = "  disabled:\n"
+    condition = "    if " + skip_if + ": " + bug_reference + "\n"
+    manifest_str = sf.wpt_add_skip_if(manifest_before1, anyjs, skip_if, bug_reference)
+    manifest_expected1 = "[myfile.html]\n" + disabled + condition + "\n"
+    assert manifest_str == manifest_expected1
+    manifest_before2 = """[myfile.html]
+  expected:
+    if fission: [OK, FAIL]
+  [< [Test 5\\] 1 out of 2 assertions were failed.]
+    expected: FAIL
+"""
+    manifest_expected2 = (
+        """[myfile.html]
+  expected:
+    if fission: [OK, FAIL]
+"""
+        + disabled
+        + condition
+        + """
+  [< [Test 5\\] 1 out of 2 assertions were failed.]
+    expected: FAIL
+"""
+    )
+    anyjs[filename] = False
+    manifest_str = sf.wpt_add_skip_if(manifest_before2, anyjs, skip_if, bug_reference)
+    assert manifest_str == manifest_expected2
+    anyjs[filename] = False
+    manifest_str = sf.wpt_add_skip_if(manifest_expected2, anyjs, skip_if, bug_reference)
+    assert manifest_str == manifest_expected2
+    manifest_before4 = """;https: //bugzilla.mozilla.org/show_bug.cgi?id=1838684
+expected: [FAIL, PASS]
+[custom-highlight-painting-overlapping-highlights-002.html]
+  expected:
+    [PASS, FAIL]
+"""
+    anyjs[filename] = False
+    manifest_str = sf.wpt_add_skip_if(manifest_before4, anyjs, skip_if, bug_reference)
+    manifest_expected4 = manifest_before4 + "\n" + manifest_expected1
+    assert manifest_str == manifest_expected4
+    manifest_before5 = """[myfile.html]
+  disabled:
+    if win11_2009 && processor == '32' && debug: Bug 456
+"""
+    anyjs[filename] = False
+    manifest_str = sf.wpt_add_skip_if(manifest_before5, anyjs, skip_if, bug_reference)
+    manifest_expected5 = manifest_before5 + condition + "\n"
+    assert manifest_str == manifest_expected5
+    manifest_before6 = """[myfile.html]
+  [Window Size]
+    expected:
+      if product == "firefox_android": FAIL
+"""
+    anyjs[filename] = False
+    manifest_str = sf.wpt_add_skip_if(manifest_before6, anyjs, skip_if, bug_reference)
+    manifest_expected6 = (
+        """[myfile.html]
+  disabled:
+"""
+        + condition
+        + """
+  [Window Size]
+    expected:
+      if product == "firefox_android": FAIL
+"""
+    )
+    assert manifest_str == manifest_expected6
+    manifest_before7 = """[myfile.html]
+  fuzzy:
+    if os == "android": maxDifference=0-255;totalPixels=0-105 # bug 1392254
+    if os == "linux": maxDifference=0-255;totalPixels=0-136 # bug 1599638
+    maxDifference=0-1;totalPixels=0-80
+  disabled:
+    if os == "win": https://bugzilla.mozilla.org/show_bug.cgi?id=1314684
+"""
+    anyjs[filename] = False
+    manifest_str = sf.wpt_add_skip_if(manifest_before7, anyjs, skip_if, bug_reference)
+    manifest_expected7 = manifest_before7 + condition + "\n"
+    assert manifest_str == manifest_expected7
+    manifest_before8 = """[myfile.html]
+  disabled:
+    if os == "linux" and os_version == "22.04" and processor == "x86_64" and debug and display == "wayland": Bug TBD
+  expected:
+    if swgl and (os == "linux") and debug and not fission: [PASS, FAIL]
+
+"""
+    anyjs[filename] = False
+    manifest_str = sf.wpt_add_skip_if(manifest_before8, anyjs, skip_if, bug_reference)
+    manifest_expected8 = (
+        """[myfile.html]
+  disabled:
+    if os == "linux" and os_version == "22.04" and processor == "x86_64" and debug and display == "wayland": Bug TBD
+"""
+        + condition
+        + """  expected:
+    if swgl and (os == "linux") and debug and not fission: [PASS, FAIL]
+
+"""
+    )
+    assert manifest_str == manifest_expected8
+    manifest_before9 = """[myfile.html]
+  disabled:
+    if os == "linux" and os_version == "22.04" and processor == "x86_64" and debug and display == "wayland": Bug TBD
+
+"""
+    anyjs[filename] = False
+    manifest_str = sf.wpt_add_skip_if(manifest_before9, anyjs, skip_if, bug_reference)
+    manifest_expected9 = (
+        """[myfile.html]
+  disabled:
+    if os == "linux" and os_version == "22.04" and processor == "x86_64" and debug and display == "wayland": Bug TBD
+"""
+        + condition
+        + "\n"
+    )
+    assert manifest_str == manifest_expected9
+    manifest_before10 = """[myfile.html]
+  disabled:
+    if os == "linux" and os_version == "22.04" and processor == "x86_64" and not debug and display == "wayland": Bug TBD
+
+  [3P fetch: Cross site window setting HTTP cookies]
+"""
+    anyjs[filename] = False
+    manifest_str = sf.wpt_add_skip_if(manifest_before10, anyjs, skip_if, bug_reference)
+    manifest_expected10 = (
+        """[myfile.html]
+  disabled:
+    if os == "linux" and os_version == "22.04" and processor == "x86_64" and not debug and display == "wayland": Bug TBD
+"""
+        + condition
+        + """\n  [3P fetch: Cross site window setting HTTP cookies]
+"""
+    )
+    assert manifest_str == manifest_expected10
 
 
 def test_get_filename_in_manifest():