Bug 1858041 - Pt.3. Temporarily skip some Tier 2 a11y_checks for unexpecredly failing/passing or crashing on Autoland. r=Jamie,extension-reviewers,devtools-reviewers,rpl,places-reviewers

There are some a11y_checks that are intermittently failing on Autoland and pass on Try and vice versa. Also, a couple of tests are crashing while performing a11y_checks, thus they are to be skipped while we continue the investigation into each of them.

Related bugs:
- Fail-if tests: meta bug 1848402
- Explore capturing click events for a11y_checks: bug 1692110

Differential Revision: https://phabricator.services.mozilla.com/D191259

browser/components/extensions/test/browser/browser.toml

@@ -277,6 +277,7 @@ fail-if = ["a11y_checks"] # Bug 1854460 clicked menuitem(s) may not be accessibl
 ["browser_ext_menus_incognito.js"]
 
 ["browser_ext_menus_refresh.js"]
+skip-if = ["a11y_checks"] # Bug 1858041 and 1835079 for causing intermittent crashes
 
 ["browser_ext_menus_replace_menu.js"]
 

browser/components/places/tests/browser/browser.toml

@@ -44,7 +44,7 @@ skip-if = ["os != 'mac'"] # Mac-only functionality
 ["browser_bookmarkProperties_speculativeConnection.js"]
 
 ["browser_bookmarkProperties_xulStore.js"]
-fail-if = ["a11y_checks"] # Bug 1854443 tree#bookmarks-view may not be labeled
+skip-if = ["a11y_checks"] # Bugs 1854443 and 1858041 to investigate intermittent a11y_checks results
 
 ["browser_bookmark_add_tags.js"]
 https_first_disabled = true

devtools/client/aboutdebugging/test/browser/browser.toml

@@ -148,6 +148,7 @@ fail-if = ["a11y_checks"] # Bug 1849028 clicked element may not be focusable and
 skip-if = ["a11y_checks"] # Bug 1849028 and 1849179 for causing crashes
 
 ["browser_aboutdebugging_fenix_runtime_node_picker.js"]
+skip-if = ["a11y_checks"] # Bug 1849028, 1858041, and 1849179 for causing crashes
 
 ["browser_aboutdebugging_hidden_addons.js"]
 
@@ -248,6 +249,7 @@ skip-if = ["a11y_checks"] # Bug 1849028 and 1849179 for causing crashes
 skip-if = ["a11y_checks"] # Bug 1849028 and 1849179 for causing crashes
 
 ["browser_aboutdebugging_telemetry_inspect.js"]
+skip-if = ["a11y_checks"] # Bug 1849028 and 1849179 for causing crashes
 
 ["browser_aboutdebugging_telemetry_navigate.js"]
 

devtools/client/debugger/test/mochitest/browser.toml

@@ -73,7 +73,7 @@ fail-if = ["a11y_checks"] # Bug 1849028 clicked element may not be focusable and
 ["browser_dbg-breakpoints-in-evaled-sources.js"]
 
 ["browser_dbg-breakpoints-list.js"]
-fail-if = ["a11y_checks"] # Bug 1849028 clicked element may not be focusable and/or labeled
+skip-if = ["a11y_checks"] # Bugs 1849028 and 1849230 for causing crashes
 
 ["browser_dbg-breakpoints-popup.js"]
 skip-if = [

devtools/client/inspector/computed/test/browser.toml

@@ -33,8 +33,10 @@ fail-if = ["a11y_checks"] # Bug 1849028 clicked element may not be focusable and
 ["browser_computed_default_tab.js"]
 
 ["browser_computed_getNodeInfo.js"]
-fail-if = ["a11y_checks"] # Bug 1849028 clicked element may not be focusable and/or labeled
-skip-if = ["!debug && os == 'mac'"] #Bug 1559033
+skip-if = [
+  "!debug && os == 'mac'", #Bug 1559033
+  "a11y_checks", # Bugs 1849028 and 1858041 to investigate intermittent a11y_checks results
+]
 
 ["browser_computed_keybindings_01.js"]
 fail-if = ["a11y_checks"] # Bug 1849028 clicked element may not be focusable and/or labeled

devtools/client/responsive/test/browser/browser.toml

@@ -47,7 +47,7 @@ skip-if = ["a11y_checks"] # Bug 1858041 and 1849028 intermittent a11y_checks res
 fail-if = ["a11y_checks"] # Bug 1849028 clicked element may not be focusable and/or labeled
 
 ["browser_device_custom_remove.js"]
-fail-if = ["a11y_checks"] # Bug 1849028 clicked element may not be focusable and/or labeled
+skip-if = ["a11y_checks"] # Bug 1849028, 1858041 to investigate intermittent a11y_checks results (fails on Try, passes on Autoland)
 
 ["browser_device_modal_exit.js"]
 

toolkit/mozapps/extensions/test/browser/browser.toml

@@ -65,8 +65,10 @@ fail-if = ["a11y_checks"] # Bug 1854646 clicked h1.header-name may not be focusa
 ["browser_checkAddonCompatibility.js"]
 
 ["browser_colorwaybuiltins_migration.js"]
-fail-if = ["a11y_checks"] # Bug 1854646 clicked addon-card may not be focusable
-skip-if = ["app-name != 'firefox'"]
+skip-if = [
+  "app-name != 'firefox'",
+  "a11y_checks", # Bugs 1854646 and 1857056 for causing crashes
+]
 
 ["browser_dragdrop.js"]
 skip-if = ["true"] # Bug 1626824

toolkit/mozapps/extensions/test/xpinstall/browser.toml

@@ -32,7 +32,7 @@ https_first_disabled = true # Bug 1737265
 
 ["browser_amosigned_trigger_iframe.js"]
 https_first_disabled = true # Bug 1737265
-skip-if = ["a11y_checks"] # Bug 1858037 to investigate intermittent a11y_checks results (fails on Try, passes on Autoland)
+skip-if = ["a11y_checks"] # Bugs 1854647 and 1858037 to investigate intermittent a11y_checks results (fails on Try, passes on Autoland)
 
 ["browser_amosigned_url.js"]
 https_first_disabled = true # Bug 1737265
@@ -46,7 +46,7 @@ https_first_disabled = true # Bug 1737265
 
 ["browser_auth4.js"]
 https_first_disabled = true # Bug 1737265
-skip-if = ["a11y_checks"] # Bug 1858037 to investigate intermittent a11y_checks results (fails on Try, passes on Autoland)
+skip-if = ["a11y_checks"] # Bugs 1854647 and 1858037 to investigate intermittent a11y_checks results (fails on Try, passes on Autoland)
 
 ["browser_badargs.js"]
 https_first_disabled = true # Bug 1737265
@@ -64,7 +64,7 @@ skip-if = ["os == 'mac' && debug"] #Bug 1590136
 
 ["browser_bug540558.js"]
 https_first_disabled = true # Bug 1737265
-skip-if = ["a11y_checks"] # Bug 1858037 to investigate intermittent a11y_checks results (fails on Try, passes on Autoland)
+skip-if = ["a11y_checks"] # Bugs 1854647 and 1858037 to investigate intermittent a11y_checks results (fails on Try, passes on Autoland)
 
 ["browser_bug611242.js"]
 
@@ -79,12 +79,13 @@ skip-if = ["true"] # disabled due to a leak. See bug 682410.
 
 ["browser_containers.js"]
 https_first_disabled = true # Bug 1737265
-fail-if = ["a11y_checks"] # Bug 1858041 clicked .popup-notification-primary-button may not be focusable
+skip-if = ["a11y_checks"] # Bugs 1854647 and 1858041 to investigate intermittent a11y_checks results (fails on Try, passes on Autoland)
 
 ["browser_cookies.js"]
 
 ["browser_cookies2.js"]
 https_first_disabled = true # Bug 1737265
+skip-if = ["a11y_checks"] # Bugs 1854647 and 1858041 to investigate intermittent a11y_checks results (fails on Autoland, passes on Try)
 
 ["browser_cookies3.js"]
 https_first_disabled = true # Bug 1737265
@@ -101,7 +102,7 @@ https_first_disabled = true # Bug 1737265
 https_first_disabled = true # Bug 1737265
 skip-if = [
   "os == 'win' && os_version == '10.0' && bits == 64", #Bug 1615449
-  "a11y_checks", # Bug 1858037 to investigate intermittent a11y_checks results (fails on Try, passes on Autoland)
+  "a11y_checks", # Bugs 1854647 and 1858037 to investigate intermittent a11y_checks results (fails on Try, passes on Autoland)
 ]
 
 ["browser_empty.js"]
@@ -110,15 +111,15 @@ skip-if = [
 
 ["browser_hash.js"]
 https_first_disabled = true # Bug 1737265
-skip-if = ["a11y_checks"] # Bug 1858037 to investigate intermittent a11y_checks results (fails on Try, passes on Autoland)
+skip-if = ["a11y_checks"] # Bugs 1854647 and 1858037 to investigate intermittent a11y_checks results (fails on Try, passes on Autoland)
 
 ["browser_hash2.js"]
 https_first_disabled = true # Bug 1737265
-skip-if = ["a11y_checks"] # Bug 1858041 to investigate intermittent a11y_checks results (fails on Autoland, passes on Try)
+skip-if = ["a11y_checks"] # Bugs 1854647 and 1858041 to investigate intermittent a11y_checks results (fails on Autoland, passes on Try)
 
 ["browser_httphash.js"]
 https_first_disabled = true # Bug 1737265
-skip-if = ["a11y_checks"] # Bug 1858041 to investigate intermittent a11y_checks results (fails on Try, passes on Autoland)
+skip-if = ["a11y_checks"] # Bugs 1854647 and 1858041 to investigate intermittent a11y_checks results (fails on Try, passes on Autoland)
 
 ["browser_httphash2.js"]
 
@@ -127,17 +128,18 @@ https_first_disabled = true # Bug 1737265
 
 ["browser_httphash4.js"]
 https_first_disabled = true # Bug 1737265
-skip-if = ["a11y_checks"] # Bug 1858037 to investigate intermittent a11y_checks results (fails on Try, passes on Autoland)
+skip-if = ["a11y_checks"] # Bugs 1854647 and 1858037 to investigate intermittent a11y_checks results (fails on Try, passes on Autoland)
 
 ["browser_httphash5.js"]
 https_first_disabled = true # Bug 1737265
+skip-if = ["a11y_checks"] # Bugs 1854647 and 1858041 to investigate intermittent a11y_checks results (fails on Autoland, passes on Try)
 
 ["browser_httphash6.js"]
 skip-if = ["true"] # Bug 1449788
 
 ["browser_installchrome.js"]
 https_first_disabled = true # Bug 1737265
-skip-if = ["a11y_checks"] # Bug 1858041 to investigate intermittent a11y_checks results (fails on Autoland, passes on Try)
+skip-if = ["a11y_checks"] # Bugs 1854647 and 1858041 to investigate intermittent a11y_checks results (fails on Autoland, passes on Try)
 
 ["browser_localfile.js"]
 
@@ -160,7 +162,7 @@ skip-if = ["debug"] # Bug 1541577 - leaks on debug
 
 ["browser_relative.js"]
 https_first_disabled = true # Bug 1737265
-skip-if = ["a11y_checks"] # Bug 1858041 to investigate intermittent a11y_checks results (fails on Try, passes on Autoland)
+skip-if = ["a11y_checks"] # Bugs 1854647 and 1858041 to investigate intermittent a11y_checks results (fails on Try, passes on Autoland)
 
 ["browser_required_useractivation.js"]
 
@@ -169,6 +171,7 @@ https_first_disabled = true # Bug 1737265
 
 ["browser_trigger_redirect.js"]
 https_first_disabled = true # Bug 1737265
+skip-if = ["a11y_checks"] # Bugs 1854647 and 1858041 to investigate intermittent a11y_checks results (fails on Autoland, passes on Try)
 
 ["browser_unsigned_trigger.js"]
 https_first_disabled = true # Bug 1737265
@@ -178,7 +181,7 @@ skip-if = ["require_signing"]
 https_first_disabled = true # Bug 1737265
 skip-if = [
   "require_signing",
-  "a11y_checks", # Bug 1858041 to investigate intermittent a11y_checks results (fails on Autoland, passes on Try)
+  "a11y_checks", # Bugs 1854647 and 1858041 to investigate intermittent a11y_checks results (fails on Autoland, passes on Try)
 ]
 
 ["browser_unsigned_trigger_xorigin.js"]
@@ -188,5 +191,5 @@ https_first_disabled = true # Bug 1737265
 https_first_disabled = true # Bug 1737265
 skip-if = [
   "require_signing",
-  "a11y_checks", # Bug 1858037 to investigate intermittent a11y_checks results (fails on Try, passes on Autoland)
+  "a11y_checks", # Bugs 1854647 and 1858037 to investigate intermittent a11y_checks results (fails on Try, passes on Autoland)
 ]