Fixing bug 379823. Allow conditional skipping of reftests (reporting them as failures). r+sr=dbaron@mozilla.com

layout/tools/reftest/README.txt

@@ -63,10 +63,20 @@ comments) must be one of the following:
       random-if(condition) The results of the test are random if a given
                            condition is met.
 
-      Examples of random-if:
-          random-if(MOZ_WIDGET_TOOLKIT=="windows")
-          random-if(MOZ_WIDGET_TOOLKIT=="cocoa")
-          random-if(MOZ_WIDGET_TOOLKIT=="gtk2") ...
+      skip  This test should not be run. This is useful when a test fails in a
+            catastrophic way, such as crashing or hanging the browser. Using
+            'skip' is prefered to simply commenting out the test because we
+            want to report the test failure at the end of the test run.
+
+      skip-if(condition) If the condition is met, the test is not run. This is
+                         useful if, for example, the test crashes only on a
+                         particular platform (i.e. it allows us to get test
+                         coverage on the other platforms).
+
+      Examples of using conditions:
+          fails-if(MOZ_WIDGET_TOOLKIT=="windows") ...
+          fails-if(MOZ_WIDGET_TOOLKIT=="cocoa") ...
+          fails-if(MOZ_WIDGET_TOOLKIT=="gtk2") ...
 
    b. <type> is one of the following:
 

layout/tools/reftest/reftest.js

@@ -58,6 +58,7 @@ var gFailureTimeout;
 const EXPECTED_PASS = 0;
 const EXPECTED_FAIL = 1;
 const EXPECTED_RANDOM = 2;
+const EXPECTED_DEATH = 3;  // test must be skipped to avoid e.g. crash/hang
 
 function OnRefTestLoad()
 {
@@ -123,16 +124,16 @@ function ReadManifest(aURL)
         var items = str.split(/\s+/); // split on whitespace
 
         var expected_status = EXPECTED_PASS;
-        while (items[0].match(/^(fails|random)/)) {
+        while (items[0].match(/^(fails|random|skip)/)) {
             var item = items.shift();
             var stat;
             var cond;
-            var m = item.match(/^(fails|random)-if(\(.*\))$/);
+            var m = item.match(/^(fails|random|skip)-if(\(.*\))$/);
             if (m) {
                 stat = m[1];
                 // Note: m[2] contains the parentheses, and we want them.
                 cond = Components.utils.evalInSandbox(m[2], sandbox);
-            } else if (item.match(/^(fails|random)$/)) {
+            } else if (item.match(/^(fails|random|skip)$/)) {
                 stat = item;
                 cond = true;
             } else {
@@ -144,6 +145,8 @@ function ReadManifest(aURL)
                     expected_status = EXPECTED_FAIL;
                 } else if (stat == "random") {
                     expected_status = EXPECTED_RANDOM;
+                } else if (stat == "skip") {
+                    expected_status = EXPECTED_DEATH;
                 }
             }
         }
@@ -167,6 +170,12 @@ function ReadManifest(aURL)
 
 function StartCurrentTest()
 {
+    // make sure we don't run tests that are expected to kill the browser
+    while (gURLs.length > 0 && gURLs[0].expected == EXPECTED_DEATH) {
+        dump("REFTEST KNOWN FAIL (SKIP): " + gURLs[0].url1.spec + "\n");
+        gURLs.shift();
+    }
+
     if (gURLs.length == 0)
         DoneTests();
     else