Bug 964646 part 7 - Handle NaN values when comparing matrices; r=dbaron

The test harness code for normalizing transform inputs to a standard form for
comparison fails to detect the case where the input is a string such as

 { tx: "20px" }

instead of:

 { tx: 20 }

When we go to compare matrix components we fail if:

  Math.abs(a.comp - b.comp) > tolerance

But if a.comp or b.comp is a string, we'll get NaN on the LHS and
"NaN > tolerance" will return false so we'll skip the failure handling and
continue onto the next component. That means if we have input { tx: "30px" } and
we get "20" as the x-translation component we'll pass the test.

This patch fixes this condition to check for isNaN.

We *could* also just drop a few .map(parseFloat) calls into
convertObjectTo3dMatrix and convertArrayTo3dMatrix to ensure "20px" becomes 20
but there may be situations where that masks bugs (since "20px" and "20em" turn
into the same thing) so for now this minimal fix should be enough.

layout/style/test/test_animations_omta.html

@@ -553,7 +553,7 @@ addAsyncTest(function *() {
   omta_is_approx("transform", { tx: 50 + 50 * gTF.ease_out(0.6) },
                  RunningOn.Compositor, 0.01, "no-0%-no-100% at 0.15s");
   advance_clock(100);
-  omta_is("transform", { tx: "100px" }, RunningOn.Compositor,
+  omta_is("transform", { tx: 100 }, RunningOn.Compositor,
           "no-0%-no-100% at 0.25s");
   advance_clock(300);
   omta_is_approx("transform", { tx: 100 - 50 * gTF.ease_out(0.4) },
@@ -958,7 +958,8 @@ function matricesRoughlyEqual(a, b, tolerance) {
   tolerance = tolerance || 0.0001;
   for (var i = 0; i < 4; i++) {
     for (var j = 0; j < 4; j++) {
-      if (Math.abs(a[i][j] - b[i][j]) > tolerance)
+      var diff = Math.abs(a[i][j] - b[i][j]);
+      if (diff > tolerance || isNaN(diff))
         return false;
     }
   }