Bug 328258. Prevent images larger than 32K pixels in height from loading to workaround Quartz bug. r=pavlov,vlad, a=beltzner

gfx/cairo/cairo/src/cairo-quartz-surface.c

@@ -39,6 +39,7 @@
 #include "cairo-quartz-private.h"
 
 #include <Carbon/Carbon.h>
+#include <limits.h>
 
 #undef QUARTZ_DEBUG
 
@@ -97,6 +98,26 @@ CG_EXTERN CGImageRef CGBitmapContextCreateImage (CGContextRef);
 static void quartz_surface_to_png (cairo_quartz_surface_t *nq, char *dest);
 static void quartz_image_to_png (CGImageRef, char *dest);
 
+/* CoreGraphics limitation with flipped CTM surfaces: height must be less than signed 16-bit max */
+
+#define CG_MAX_HEIGHT   SHRT_MAX
+#define CG_MAX_WIDTH    USHRT_MAX
+
+/* is the desired size of the surface within bounds? */
+static cairo_bool_t verify_surface_size(int width, int height)
+{
+    /* hmmm, allow width, height == 0 ? */
+    if (width < 0 || height < 0) {
+	return FALSE;
+    }
+
+    if (width > CG_MAX_WIDTH || height > CG_MAX_HEIGHT) {
+	return FALSE;
+    }
+
+    return TRUE;
+}
+
 /*
  * Cairo path -> Quartz path conversion helpers
  */
@@ -981,6 +1002,12 @@ _cairo_quartz_surface_create_similar (void *abstract_surface,
 	format = CAIRO_FORMAT_A8;
     else
 	return NULL;
+	
+    // verify width and height of surface
+    if (!verify_surface_size(width, height)) {
+	_cairo_error (CAIRO_STATUS_NO_MEMORY);
+	return NULL;
+    }
 
     return cairo_quartz_surface_create (format, width, height);
 }
@@ -996,6 +1023,13 @@ _cairo_quartz_surface_clone_similar (void *abstract_surface,
 {
     cairo_quartz_surface_t *new_surface = NULL;
     cairo_format_t new_format;
+    
+    *clone_out = NULL;
+
+    // verify width and height of surface
+    if (!verify_surface_size(width, height)) {
+	return CAIRO_INT_STATUS_UNSUPPORTED;
+    }
 
     CGImageRef quartz_image = NULL;
 
@@ -1032,6 +1066,7 @@ _cairo_quartz_surface_clone_similar (void *abstract_surface,
 
 	new_format = isurf->format;
 
+        
 	dataProvider = CGDataProviderCreateWithData (NULL,
 						     isurf->data,
 						     isurf->height * isurf->stride,
@@ -1681,6 +1716,12 @@ cairo_quartz_surface_create (cairo_format_t format,
     int stride;
     int bitsPerComponent;
 
+    // verify width and height of surface
+    if (!verify_surface_size(width, height)) {
+	_cairo_error (CAIRO_STATUS_NO_MEMORY);
+	return (cairo_surface_t*) &_cairo_surface_nil;
+    }
+
     if (format == CAIRO_FORMAT_ARGB32) {
 	cgColorspace = CGColorSpaceCreateDeviceRGB();
 	stride = width * 4;

gfx/src/shared/gfxImageFrame.cpp

@@ -39,6 +39,7 @@
 
 #include "gfxImageFrame.h"
 #include "nsIServiceManager.h"
+#include <limits.h>
 
 NS_IMPL_ISUPPORTS2(gfxImageFrame, gfxIImageFrame, nsIInterfaceRequestor)
 
@@ -63,6 +64,8 @@ NS_IMETHODIMP gfxImageFrame::Init(PRInt32 aX, PRInt32 aY, PRInt32 aWidth, PRInt3
   if (mInitialized)
     return NS_ERROR_FAILURE;
 
+  // assert for properties that should be verified by decoders, warn for properties related to bad content
+  
   if (aWidth <= 0 || aHeight <= 0) {
     NS_ASSERTION(0, "error - negative image size\n");
     return NS_ERROR_FAILURE;
@@ -71,26 +74,34 @@ NS_IMETHODIMP gfxImageFrame::Init(PRInt32 aX, PRInt32 aY, PRInt32 aWidth, PRInt3
   /* check to make sure we don't overflow a 32-bit */
   PRInt32 tmp = aWidth * aHeight;
   if (tmp / aHeight != aWidth) {
-    NS_ASSERTION(0, "width or height too large\n");
+    NS_WARNING("width or height too large");
     return NS_ERROR_FAILURE;
   }
   tmp = tmp * 4;
   if (tmp / 4 != aWidth * aHeight) {
-    NS_ASSERTION(0, "width or height too large\n");
+    NS_WARNING("width or height too large");
     return NS_ERROR_FAILURE;
   }
 
   if ( (aDepth != 8) && (aDepth != 24) ){
-    NS_ERROR("This Depth is not supported\n");
+    NS_ERROR("This Depth is not supported");
     return NS_ERROR_FAILURE;
   }
 
   /* reject over-wide or over-tall images */
   const PRInt32 k64KLimit = 0x0000FFFF;
   if ( aWidth > k64KLimit || aHeight > k64KLimit ){
-    NS_ERROR("image too big");
+    NS_WARNING("image too big");
     return NS_ERROR_FAILURE;
   }
+  
+#if defined(XP_MACOSX)
+  // CoreGraphics is limited to images < 32K in *height*, so clamp all surfaces on the Mac to that height
+  if (aHeight > SHRT_MAX) {
+    NS_WARNING("image too big");
+    return NS_ERROR_FAILURE;
+  }
+#endif
 
   nsresult rv;
 

gfx/thebes/src/gfxASurface.cpp

@@ -55,6 +55,7 @@
 #endif
 
 #include <stdio.h>
+#include <limits.h>
 
 static cairo_user_data_key_t gfxasurface_pointer_key;
 
@@ -281,6 +282,14 @@ gfxASurface::CheckSurfaceSize(const gfxIntSize& sz, PRInt32 limit)
         return PR_FALSE;
     }
 
+#if defined(XP_MACOSX)
+    // CoreGraphics is limited to images < 32K in *height*, so clamp all surfaces on the Mac to that height
+    if (sz.height > SHRT_MAX) {
+        NS_WARNING("Surface size too large (would overflow)!");
+        return PR_FALSE;
+    }
+#endif
+
     // check to make sure we don't overflow a 32-bit
     PRInt32 tmp = sz.width * sz.height;
     if (tmp && tmp / sz.height != sz.width) {

modules/libpr0n/decoders/bmp/nsBMPDecoder.cpp

@@ -122,17 +122,32 @@ NS_IMETHODIMP nsBMPDecoder::Flush()
 
 NS_METHOD nsBMPDecoder::ReadSegCb(nsIInputStream* aIn, void* aClosure,
                              const char* aFromRawSegment, PRUint32 aToOffset,
-                             PRUint32 aCount, PRUint32 *aWriteCount) {
+                             PRUint32 aCount, PRUint32 *aWriteCount) 
+{
     nsBMPDecoder *decoder = reinterpret_cast<nsBMPDecoder*>(aClosure);
     *aWriteCount = aCount;
-    return decoder->ProcessData(aFromRawSegment, aCount);
+    
+    nsresult rv = decoder->ProcessData(aFromRawSegment, aCount);
+
+    if (NS_FAILED(rv)) {
+        *aWriteCount = 0;
+    }
+
+    return rv;
 }
 
 NS_IMETHODIMP nsBMPDecoder::WriteFrom(nsIInputStream *aInStr, PRUint32 aCount, PRUint32 *aRetval)
 {
     PR_LOG(gBMPLog, PR_LOG_DEBUG, ("nsBMPDecoder::WriteFrom(%p, %lu, %p)\n", aInStr, aCount, aRetval));
 
-    return aInStr->ReadSegments(ReadSegCb, this, aCount, aRetval);
+    nsresult rv = aInStr->ReadSegments(ReadSegCb, this, aCount, aRetval);
+    
+    if (aCount != *aRetval) { 
+        *aRetval = aCount; 
+        return NS_ERROR_FAILURE; 
+    }
+    
+    return rv;    
 }
 
 // ----------------------------------------

modules/libpr0n/decoders/gif/nsGIFDecoder2.cpp

@@ -965,6 +965,12 @@ nsresult nsGIFDecoder2::GifWrite(const PRUint8 *buf, PRUint32 len)
       }
 
       BeginImageFrame();
+      
+      // handle allocation error
+      if (!mImageFrame) {
+        mGIFStruct.state = gif_error;
+        break;
+      }
 
       if (q[8] & 0x40) {
         mGIFStruct.interlaced = PR_TRUE;
@@ -1055,11 +1061,15 @@ nsresult nsGIFDecoder2::GifWrite(const PRUint8 *buf, PRUint32 len)
       break;
 
     case gif_done:
-    case gif_error:
       EndGIF();
       return NS_OK;
       break;
 
+    case gif_error:
+      EndGIF();
+      return NS_ERROR_FAILURE;
+      break;
+
     // Handle out of memory errors
     case gif_oom:
       return NS_ERROR_OUT_OF_MEMORY;
@@ -1070,6 +1080,12 @@ nsresult nsGIFDecoder2::GifWrite(const PRUint8 *buf, PRUint32 len)
     }
   }
 
+  // if an error state is set but no data remains, code flow reaches here
+  if (mGIFStruct.state == gif_error) {
+      EndGIF();
+      return NS_ERROR_FAILURE;
+  }
+  
   // Copy the leftover into mGIFStruct.hold
   mGIFStruct.bytes_in_hold = len;
   if (len) {

modules/libpr0n/decoders/xbm/nsXBMDecoder.cpp

@@ -121,12 +121,26 @@ NS_METHOD nsXBMDecoder::ReadSegCb(nsIInputStream* aIn, void* aClosure,
                              PRUint32 aCount, PRUint32 *aWriteCount) {
     nsXBMDecoder *decoder = reinterpret_cast<nsXBMDecoder*>(aClosure);
     *aWriteCount = aCount;
-    return decoder->ProcessData(aFromRawSegment, aCount);
+
+    nsresult rv = decoder->ProcessData(aFromRawSegment, aCount);
+
+    if (NS_FAILED(rv)) {
+        *aWriteCount = 0;
+    }
+
+    return rv;
 }
 
 NS_IMETHODIMP nsXBMDecoder::WriteFrom(nsIInputStream *aInStr, PRUint32 aCount, PRUint32 *aRetval)
 {
-    return aInStr->ReadSegments(ReadSegCb, this, aCount, aRetval);
+    nsresult rv = aInStr->ReadSegments(ReadSegCb, this, aCount, aRetval);
+    
+    if (aCount != *aRetval) { 
+        *aRetval = aCount; 
+        return NS_ERROR_FAILURE; 
+    }
+    
+    return rv;    
 }
 
 nsresult nsXBMDecoder::ProcessData(const char* aData, PRUint32 aCount) {