Bug 83804 - crash in gif decoder caused by mismatch in rgb line buffer size.

Moved ownership of rgb line buffer to nsGIFDecoder2.
r=pavlov, sr=blizzard, a=blizzard

modules/libpr0n/decoders/gif/GIF2.cpp

@@ -198,7 +198,6 @@ output_row(gif_struct *gs)
             int result = (gs->GIFCallback_HaveDecodedRow)(
               gs->clientptr,
               gs->rowbuf,                /* Pointer to single scanline temporary buffer */
-              gs->rgbrow,                /* Pointer to temporary storage for dithering/mapping */
               gs->x_offset,              /* x offset with respect to GIF logical screen origin */
               width,                     /* Length of the row */
               drow_start,                /* Row number */
@@ -478,7 +477,6 @@ PRBool GIFInit(
   int (*PR_CALLBACK GIFCallback_HaveDecodedRow)(
     void* aClientData,
     PRUint8* aRowBufPtr,   /* Pointer to single scanline temporary buffer */
-    PRUint8* aRGBrowBufPtr,/* Pointer to temporary storage for dithering/mapping */
     int aXOffset,          /* With respect to GIF logical screen origin */
     int aLength,           /* Length of the row? */
     int aRow,              /* Row number? */
@@ -1260,11 +1258,9 @@ int gif_write(gif_struct *gs, const PRUint8 *buf, PRUint32 len)
             if (gs->screen_width < width) {
                 /* XXX Deviant! */
 
-                //gs->rgbrow = (PRUint8*)PR_REALLOC(gs->rgbrow, 3 * width);
-                gs->rgbrow = (PRUint8*)PR_REALLOC(gs->rgbrow, sizeof(GIF_RGB) * width);
                 gs->rowbuf = (PRUint8*)PR_REALLOC(gs->rowbuf, width);
 
-                if((!gs->rgbrow)||(!gs->rowbuf)){
+                if(!gs->rowbuf){
                   gs->state = gif_oom;
                   break;
                 }
@@ -1300,15 +1296,11 @@ int gif_write(gif_struct *gs, const PRUint8 *buf, PRUint32 len)
                 //}
         }
         else{
-            if (!gs->rgbrow)
-                gs->rgbrow = (PRUint8*)PR_MALLOC(sizeof(GIF_RGB) * gs->screen_width);
-                //gs->rgbrow = (PRUint8*)PR_MALLOC(3 * gs->screen_width);
-
             if (!gs->rowbuf)
                 gs->rowbuf = (PRUint8*)PR_MALLOC(gs->screen_width);
         }
 
-            if (!gs->rowbuf || !gs->rgbrow)
+            if (!gs->rowbuf)
             {
                     //ILTRACE(0,("il:gif: MEM row"));
                     gs->state=gif_oom;
@@ -1674,7 +1666,6 @@ gif_destroy(gif_struct *gs)
   gif_destroy_transparency(gs);
 
   PR_FREEIF(gs->rowbuf);
-  PR_FREEIF(gs->rgbrow);
   PR_FREEIF(gs->prefix);
   PR_FREEIF(gs->suffix);
   PR_FREEIF(gs->stack);

modules/libpr0n/decoders/gif/GIF2.h

@@ -131,7 +131,6 @@ typedef struct gif_struct {
     int (PR_CALLBACK* GIFCallback_HaveDecodedRow)(
       void* aClientData,
       PRUint8* aRowBufPtr,   /* Pointer to single scanline temporary buffer */
-      PRUint8* aRGBrowBufPtr,/* Pointer to temporary storage for dithering/mapping */
       int aXOffset,          /* With respect to GIF logical screen origin */
       int aLength,           /* Length of the row? */
       int aRow,              /* Row number? */
@@ -171,7 +170,6 @@ typedef struct gif_struct {
     int ipass;                  /* Interlace pass; Ranges 1-4 if interlaced. */
     PRUintn rows_remaining;        /* Rows remaining to be output */
     PRUintn irow;                  /* Current output row, starting at zero */
-    PRUint8 *rgbrow;              /* Temporary storage for dithering/mapping */
     PRUint8 *rowbuf;              /* Single scanline temporary buffer */
     PRUint8 *rowend;              /* Pointer to end of rowbuf */
     PRUint8 *rowp;                /* Current output pointer */
@@ -257,7 +255,6 @@ PRBool GIFInit(
   int (*PR_CALLBACK GIFCallback_HaveDecodedRow)(
     void* aClientData,
     PRUint8* aRowBufPtr,   /* Pointer to single scanline temporary buffer */
-    PRUint8* aRGBrowBufPtr,/* Pointer to temporary storage for dithering/mapping */
     int aXOffset,          /* With respect to GIF logical screen origin */
     int aLength,           /* Length of the row? */
     int aRow,              /* Row number? */

modules/libpr0n/decoders/gif/nsGIFDecoder2.cpp

@@ -45,6 +45,7 @@ nsGIFDecoder2::nsGIFDecoder2()
   mGIFStruct = nsnull;
 
   mAlphaLine = nsnull;
+  mRGBLine = nsnull;
   mBackgroundRGBIndex = 0;
 
   mCurrentRow = -1;
@@ -59,6 +60,9 @@ nsGIFDecoder2::~nsGIFDecoder2(void)
   if (mAlphaLine)
     nsMemory::Free(mAlphaLine);
 
+  if (mRGBLine)
+    nsMemory::Free(mRGBLine);
+
   if (mGIFStruct) {
     gif_destroy(mGIFStruct);
     mGIFStruct = nsnull;
@@ -378,7 +382,6 @@ int HaveImageAll(
 int HaveDecodedRow(
   void* aClientData,
   PRUint8* aRowBufPtr,   // Pointer to single scanline temporary buffer
-  PRUint8* aRGBrowBufPtr,// Pointer to temporary storage for dithering/mapping
   int aXOffset,          // With respect to GIF logical screen origin
   int aLength,           // Length of the row?
   int aRowNumber,        // Row number?
@@ -418,10 +421,10 @@ int HaveDecodedRow(
     decoder->mImageFrame->GetImageBytesPerRow(&bpr);
     decoder->mImageFrame->GetAlphaBytesPerRow(&abpr);
 
+    decoder->mRGBLine = (PRUint8 *)nsMemory::Realloc(decoder->mRGBLine, bpr);
+
     if (format == gfxIFormats::RGB_A1 || format == gfxIFormats::BGR_A1) {
-      if (decoder->mAlphaLine)
-        nsMemory::Free(decoder->mAlphaLine);
-      decoder->mAlphaLine = (PRUint8 *)nsMemory::Alloc(abpr);
+      decoder->mAlphaLine = (PRUint8 *)nsMemory::Realloc(decoder->mAlphaLine, abpr);
     }
   } else {
     decoder->mImageFrame->GetImageBytesPerRow(&bpr);
@@ -456,7 +459,7 @@ int HaveDecodedRow(
       cmap = decoder->mGIFStruct->local_colormap;
     }
 
-    PRUint8* rgbRowIndex = aRGBrowBufPtr;
+    PRUint8* rgbRowIndex = decoder->mRGBLine;
     PRUint8* rowBufIndex = aRowBufPtr;
         
     switch (format) {
@@ -473,7 +476,7 @@ int HaveDecodedRow(
         }
 
         for (int i=0; i<aDuplicateCount; i++)
-          decoder->mImageFrame->SetImageData((PRUint8*)aRGBrowBufPtr,
+          decoder->mImageFrame->SetImageData(decoder->mRGBLine,
                                              bpr, (aRowNumber+i)*bpr);
       }
       break;
@@ -487,7 +490,7 @@ int HaveDecodedRow(
         }
 
         for (int i=0; i<aDuplicateCount; i++)
-          decoder->mImageFrame->SetImageData((PRUint8*)aRGBrowBufPtr,
+          decoder->mImageFrame->SetImageData(decoder->mRGBLine,
                                              bpr, (aRowNumber+i)*bpr);
       }
       break;
@@ -502,7 +505,7 @@ int HaveDecodedRow(
             decoder->mImageFrame->SetTransparentColor(transColor);
         }
 
-        memset(aRGBrowBufPtr, 0, bpr);
+        memset(decoder->mRGBLine, 0, bpr);
         memset(decoder->mAlphaLine, 0, abpr);
         PRUint32 iwidth = (PRUint32)width;
         for (PRUint32 x=0; x<iwidth; x++) {
@@ -531,7 +534,7 @@ int HaveDecodedRow(
           ++rowBufIndex;
         }
         for (int i=0; i<aDuplicateCount; i++) {
-          decoder->mImageFrame->SetImageData((PRUint8*)aRGBrowBufPtr,
+          decoder->mImageFrame->SetImageData(decoder->mRGBLine,
                                              bpr, (aRowNumber+i)*bpr);
           decoder->mImageFrame->SetAlphaData(decoder->mAlphaLine,
                                              abpr, (aRowNumber+i)*abpr);

modules/libpr0n/decoders/gif/nsGIFDecoder2.h

@@ -70,6 +70,7 @@ public:
   gif_struct *mGIFStruct;
   
   PRUint8 *mAlphaLine;
+  PRUint8 *mRGBLine;
   PRUint8 mBackgroundRGBIndex;
   PRUint8 mCurrentPass;
   PRUint8 mLastFlushedPass;
@@ -85,7 +86,6 @@ static int PR_CALLBACK BeginGIF(
 static int PR_CALLBACK HaveDecodedRow(
   void* aClientData,
   PRUint8* aRowBufPtr,   // Pointer to single scanline temporary buffer
-  PRUint8* aRGBrowBufPtr,// Pointer to temporary storage for dithering/mapping
   int aXOffset,          // With respect to GIF logical screen origin
   int aLength,           // Length of the row?
   int aRow,              // Row number?