Bug 1678938 - nudge clipped coordinates inside clip boundary. r=jimb

Differential Revision: https://phabricator.services.mozilla.com/D99008

gfx/tests/crashtests/1678938-1.html

@@ -0,0 +1,9 @@
+<style>
+:first-of-type {
+  filter: saturate(70%);
+  transform: skew(-224.96198773774648grad) matrix3d(79.00629819492802, -0.0, 43.223526565331326, -294.8479790240964, 269.7, 243.87676190037186, 238.83225166324632, 42.507227157006135, 634943.1, 208.66200905121616, 46.19831959954935, -285.43139932334543, -229.39776691490985, -126.46021751791264, 116.46, 137.77);
+  border-block-end-style: double;
+</style>
+<dl>
+<header>
+</html>

gfx/tests/crashtests/crashtests.list

@@ -198,4 +198,6 @@ load 1650989-very-large-mask.html
 load 1650990.html
 skip-if(!webrender||AddressSanitizer) load 1652750-deep-scene-stack.html
 load 1651882.html
+skip-if(!webrender) load 1678938-1.html
 load 1679477-1.html
+

gfx/wr/swgl/src/gl.cc

@@ -3748,7 +3748,13 @@ static int clip_side(int nump, Point3D* p, Interpolants* interp, Point3D* outP,
         assert(numClip < nump + 2);
         float prevDist = prevCoord - prevSide * prev.w;
         float curDist = curCoord - prevSide * cur.w;
-        float k = prevDist / (prevDist - curDist);
+        // It may happen that after we interpolate by the weight k that due to
+        // floating point rounding we've underestimated the value necessary to
+        // push it over the clipping boundary. Just in case, nudge the mantissa
+        // by a single increment so that we essentially round it up and move it
+        // further inside the clipping boundary. We use nextafter to do this in
+        // a portable fashion.
+        float k = nextafterf(prevDist / (prevDist - curDist), 1.0f);
         outP[numClip] = prev + (cur - prev) * k;
         outInterp[numClip] = prevInterp + (curInterp - prevInterp) * k;
         numClip++;
@@ -3760,7 +3766,9 @@ static int clip_side(int nump, Point3D* p, Interpolants* interp, Point3D* outP,
         assert(numClip < nump + 2);
         float prevDist = prevCoord - curSide * prev.w;
         float curDist = curCoord - curSide * cur.w;
-        float k = prevDist / (prevDist - curDist);
+        // Calculate interpolation weight k and the nudge it inside clipping
+        // boundary with nextafter.
+        float k = nextafterf(prevDist / (prevDist - curDist), 1.0f);
         outP[numClip] = prev + (cur - prev) * k;
         outInterp[numClip] = prevInterp + (curInterp - prevInterp) * k;
         numClip++;