245631 patch by Son Le <son.le0@gmail.com> r=biesi sr=dveditz a=asa

- Return error from WriteFrom when ProcessData fails
- verify that the type of the icon is correct (icon or cursor)
- make sure that the image offset is valid
- check for OOM (mRow, mAlphaBuffer)
- add safety checks for mRow and mAlphaBuffer

modules/libpr0n/decoders/bmp/nsICODecoder.cpp

@@ -142,6 +142,7 @@ nsICODecoder::nsICODecoder()
 {
   mPos = mNumColors = mRowBytes = mImageOffset = mCurrIcon = mNumIcons = 0;
   mCurLine = 1; // Otherwise decoder will never start
+  mStatus = NS_OK;
   mColors = nsnull;
   mRow = nsnull;
   mHaveAlphaData = 0;
@@ -215,14 +216,17 @@ NS_IMETHODIMP nsICODecoder::Flush()
 NS_METHOD nsICODecoder::ReadSegCb(nsIInputStream* aIn, void* aClosure,
                              const char* aFromRawSegment, PRUint32 aToOffset,
                              PRUint32 aCount, PRUint32 *aWriteCount) {
-    nsICODecoder *decoder = NS_REINTERPRET_CAST(nsICODecoder*, aClosure);
-    *aWriteCount = aCount;
-    return decoder->ProcessData(aFromRawSegment, aCount);
+  nsICODecoder *decoder = NS_REINTERPRET_CAST(nsICODecoder*, aClosure);
+  *aWriteCount = aCount;
+  decoder->mStatus = decoder->ProcessData(aFromRawSegment, aCount);
+  return decoder->mStatus;
 }
 
 NS_IMETHODIMP nsICODecoder::WriteFrom(nsIInputStream *aInStr, PRUint32 aCount, PRUint32 *aRetval)
 {
-    return aInStr->ReadSegments(ReadSegCb, this, aCount, aRetval);
+  nsresult rv = aInStr->ReadSegments(ReadSegCb, this, aCount, aRetval);
+  NS_ENSURE_SUCCESS(rv, rv);
+  return mStatus;
 }
 
 nsresult nsICODecoder::ProcessData(const char* aBuffer, PRUint32 aCount) {
@@ -230,8 +234,12 @@ nsresult nsICODecoder::ProcessData(const char* aBuffer, PRUint32 aCount) {
     return NS_OK;
 
   while (aCount && (mPos < ICONCOUNTOFFSET)) { // Skip to the # of icons.
-    if (mPos == 2) // if the third byte is 2: This is a cursor
+    if (mPos == 2) { // if the third byte is 1: This is an icon, 2: a cursor
+      if ((*aBuffer != 1) && (*aBuffer != 2)) {
+        return NS_ERROR_FAILURE;
+      }
       mIsCursor = (*aBuffer == 2);
+    }
     mPos++; aBuffer++; aCount--;
   }
 
@@ -267,6 +275,12 @@ nsresult nsICODecoder::ProcessData(const char* aBuffer, PRUint32 aCount) {
       if ((e.mWidth == PREFICONSIZE && e.mHeight == PREFICONSIZE && e.mBitCount >= colorDepth)
            || (mCurrIcon == mNumIcons && mImageOffset == 0)) {
         mImageOffset = e.mImageOffset;
+
+        // ensure mImageOffset is >= the size of the direntry headers (bug #245631)
+        PRUint32 minImageOffset = DIRENTRYOFFSET + mNumIcons*sizeof(mDirEntryArray);
+        if (mImageOffset < minImageOffset)
+          return NS_ERROR_FAILURE;
+
         colorDepth = e.mBitCount;
         memcpy(&mDirEntry, &e, sizeof(IconDirEntry));
       }
@@ -396,6 +410,13 @@ nsresult nsICODecoder::ProcessData(const char* aBuffer, PRUint32 aCount) {
         return NS_ERROR_OUT_OF_MEMORY;
     }
 
+    // Ensure memory has been allocated before decoding. If we get this far 
+    // without allocated memory, the file is most likely invalid.
+    NS_ASSERTION(mRow, "mRow is null");
+    NS_ASSERTION(mDecodedBuffer, "mDecodedBuffer is null");
+    if (!mRow || !mDecodedBuffer)
+      return NS_ERROR_FAILURE;
+
     PRUint32 rowSize = (mBIH.bpp * mDirEntry.mWidth + 7) / 8; // +7 to round up
     if (rowSize % 4)
       rowSize += (4 - (rowSize % 4)); // Pad to DWORD Boundary
@@ -491,10 +512,20 @@ nsresult nsICODecoder::ProcessData(const char* aBuffer, PRUint32 aCount) {
       mCurLine = mDirEntry.mHeight;
       delete []mRow;
       mRow = new PRUint8[rowSize];
+      if (!mRow)
+        return NS_ERROR_OUT_OF_MEMORY;
       mAlphaBuffer = new PRUint8[mDirEntry.mHeight*rowSize];
+      if (!mAlphaBuffer)
+        return NS_ERROR_OUT_OF_MEMORY;
       memset(mAlphaBuffer, 0xff, mDirEntry.mHeight*rowSize);
     }
 
+    // Ensure memory has been allocated before decoding.
+    NS_ASSERTION(mRow, "mRow is null");
+    NS_ASSERTION(mAlphaBuffer, "mAlphaBuffer is null");
+    if (!mRow || !mAlphaBuffer)
+      return NS_ERROR_FAILURE;
+
     PRUint32 toCopy;
     do {
         if (mCurLine == 0) {

modules/libpr0n/decoders/bmp/nsICODecoder.h

@@ -127,6 +127,8 @@ private:
   PRUint32 mRowBytes; // How many bytes of the row were already received
   PRInt32 mCurLine;
 
+  nsresult mStatus;
+
   PRUint8* mDecodedBuffer;
   PRUint8* mAlphaBuffer;
   PRPackedBool mIsCursor;