bug 376573 prevent memory leak if realloc fails. r=biesi, sr=stuart

modules/libpr0n/decoders/xbm/nsXBMDecoder.cpp

@@ -136,15 +136,27 @@ nsresult nsXBMDecoder::ProcessData(const char* aData, PRUint32 aCount) {
     // calculate the offset since the absolute position might no longer
     // be valid after realloc
     const PRPtrdiff posOffset = mPos ? (mPos - mBuf) : 0;
-    mBuf = (char*)realloc(mBuf, mBufSize + aCount + 1);
+
+    // expand the buffer to hold the new data
+    char* oldbuf = mBuf;
+    PRUint32 newbufsize = mBufSize + aCount + 1;
+    if (newbufsize < mBufSize)
+        mBuf = nsnull;  // size wrapped around, give up
+    else
+        mBuf = (char*)realloc(mBuf, newbufsize);
+
     if (!mBuf) {
         mState = RECV_DONE;
+        if (oldbuf)
+            free(oldbuf);
         return NS_ERROR_OUT_OF_MEMORY;
     }
     memcpy(mBuf + mBufSize, aData, aCount);
     mBufSize += aCount;
     mBuf[mBufSize] = 0;
     mPos = mBuf + posOffset;
+
+    // process latest data according to current state
     if (mState == RECV_HEADER) {
         mPos = strstr(mBuf, "#define");
         if (!mPos)