Bug 1099437 - Part 1: Protect against negative sizes and overflow. r=nical

gfx/layers/ImageDataSerializer.cpp

@@ -8,6 +8,7 @@
 #include "gfxPoint.h"                   // for gfxIntSize
 #include "mozilla/Assertions.h"         // for MOZ_ASSERT, etc
 #include "mozilla/gfx/2D.h"             // for DataSourceSurface, Factory
+#include "mozilla/gfx/Logging.h"        // for gfxDebug
 #include "mozilla/gfx/Tools.h"          // for GetAlignedStride, etc
 #include "mozilla/mozalloc.h"           // for operator delete, etc
 
@@ -68,16 +69,36 @@ ImageDataSerializer::InitializeBufferInfo(IntSize aSize,
 static inline uint32_t
 ComputeStride(SurfaceFormat aFormat, uint32_t aWidth)
 {
-  return GetAlignedStride<4>(BytesPerPixel(aFormat) * aWidth);
+  CheckedInt<uint32_t> size = BytesPerPixel(aFormat);
+  size *= aWidth;
+  if (!size.isValid() || size == 0) {
+    gfxDebug() << "ComputeStride overflow " << aWidth;
+    return 0;
+  }
+
+  return GetAlignedStride<4>(size.value());
 }
 
 uint32_t
 ImageDataSerializerBase::ComputeMinBufferSize(IntSize aSize,
-                                          SurfaceFormat aFormat)
+                                              SurfaceFormat aFormat)
 {
-  uint32_t bufsize = aSize.height * ComputeStride(aFormat, aSize.width);
+  MOZ_ASSERT(aSize.height >= 0 && aSize.width >= 0);
+  if (aSize.height <= 0 || aSize.width <= 0) {
+    gfxDebug() << "Non-positive image buffer size request " << aSize.width << "x" << aSize.height;
+    return 0;
+  }
+
+  CheckedInt<uint32_t> bufsize = ComputeStride(aFormat, aSize.width);
+  bufsize *= aSize.height;
+
+  if (!bufsize.isValid() || bufsize.value() == 0) {
+    gfxDebug() << "Buffer size overflow " << aSize.width << "x" << aSize.height;
+    return 0;
+  }
+
   return SurfaceBufferInfo::GetOffset()
-       + GetAlignedStride<16>(bufsize);
+       + GetAlignedStride<16>(bufsize.value());
 }
 
 void

gfx/layers/YCbCrImageDataSerializer.cpp

@@ -8,6 +8,7 @@
 #include "gfx2DGlue.h"                  // for ToIntSize
 #include "mozilla/gfx/2D.h"             // for DataSourceSurface, Factory
 #include "mozilla/gfx/BaseSize.h"       // for BaseSize
+#include "mozilla/gfx/Logging.h"        // for gfxDebug
 #include "mozilla/gfx/Types.h"
 #include "mozilla/mozalloc.h"           // for operator delete
 #include "yuv_convert.h"                // for ConvertYCbCrToRGB32, etc
@@ -73,7 +74,7 @@ void YCbCrImageDataDeserializerBase::Validate()
                           info->mYStride,
                           IntSize(info->mCbCrWidth, info->mCbCrHeight),
                           info->mCbCrStride);
-  mIsValid = requiredSize <= mDataSize;
+  mIsValid = requiredSize && requiredSize <= mDataSize;
 
 }
 
@@ -140,10 +141,15 @@ static size_t ComputeOffset(uint32_t aHeight, uint32_t aStride)
 // Minimum required shmem size in bytes
 size_t
 YCbCrImageDataDeserializerBase::ComputeMinBufferSize(const gfx::IntSize& aYSize,
-                                                   uint32_t aYStride,
-                                                   const gfx::IntSize& aCbCrSize,
-                                                   uint32_t aCbCrStride)
+                                                     uint32_t aYStride,
+                                                     const gfx::IntSize& aCbCrSize,
+                                                     uint32_t aCbCrStride)
 {
+  MOZ_ASSERT(aYSize.height >= 0 && aYSize.width >= 0);
+  if (aYSize.height <= 0 || aYSize.width <= 0 || aCbCrSize.height <= 0 || aCbCrSize.width <= 0) {
+    gfxDebug() << "Non-positive YCbCr buffer size request " << aYSize.height << "x" << aYSize.width << ", " << aCbCrSize.height << "x" << aCbCrSize.width;
+    return 0;
+  }
   return ComputeOffset(aYSize.height, aYStride)
          + 2 * ComputeOffset(aCbCrSize.height, aCbCrStride)
          + MOZ_ALIGN_WORD(sizeof(YCbCrBufferInfo));

gfx/layers/basic/TextureClientX11.cpp

@@ -8,6 +8,7 @@
 #include "mozilla/layers/ISurfaceAllocator.h"
 #include "mozilla/layers/ShadowLayerUtilsX11.h"
 #include "mozilla/gfx/2D.h"
+#include "mozilla/gfx/Logging.h"
 #include "gfxXlibSurface.h"
 #include "gfx2DGlue.h"
 
@@ -38,6 +39,8 @@ TextureClientX11::CreateSimilar(TextureFlags aFlags,
 {
   RefPtr<TextureClient> tex = new TextureClientX11(mAllocator, mFormat, mFlags);
 
+  // mSize is guaranteed to be non-negative
+  MOZ_ASSERT(mSize.width >= 0 && mSize.height >= 0);
   if (!tex->AllocateForSurface(mSize, aAllocFlags)) {
     return nullptr;
   }
@@ -107,6 +110,11 @@ TextureClientX11::AllocateForSurface(IntSize aSize, TextureAllocationFlags aText
   MOZ_ASSERT(!IsAllocated());
   //MOZ_ASSERT(mFormat != gfx::FORMAT_YUV, "This TextureClient cannot use YCbCr data");
 
+  MOZ_ASSERT(aSize.width >= 0 && aSize.height >= 0);
+  if (aSize.width <= 0 || aSize.height <= 0) {
+    gfxDebug() << "Asking for X11 surface of invalid size " << aSize.width << "x" << aSize.height;
+    return false;
+  }
   gfxContentType contentType = ContentForFormat(mFormat);
   nsRefPtr<gfxASurface> surface = gfxPlatform::GetPlatform()->CreateOffscreenSurface(aSize, contentType);
   if (!surface || surface->GetType() != gfxSurfaceType::Xlib) {

gfx/layers/client/TextureClient.cpp

@@ -17,6 +17,7 @@
 #include "nsISupportsImpl.h"            // for MOZ_COUNT_CTOR, etc
 #include "ImageContainer.h"             // for PlanarYCbCrData, etc
 #include "mozilla/gfx/2D.h"
+#include "mozilla/gfx/Logging.h"        // for gfxDebug
 #include "mozilla/layers/TextureClientOGL.h"
 #include "mozilla/layers/PTextureChild.h"
 #include "SharedSurface.h"
@@ -584,8 +585,10 @@ bool
 ShmemTextureClient::Allocate(uint32_t aSize)
 {
   MOZ_ASSERT(mValid);
-  SharedMemory::SharedMemoryType memType = OptimalShmemType();
-  mAllocated = GetAllocator()->AllocUnsafeShmem(aSize, memType, &mShmem);
+  if (aSize > 0) {
+    SharedMemory::SharedMemoryType memType = OptimalShmemType();
+    mAllocated = GetAllocator()->AllocUnsafeShmem(aSize, memType, &mShmem);
+  }
   return mAllocated;
 }
 
@@ -716,8 +719,12 @@ BufferTextureClient::AllocateForSurface(gfx::IntSize aSize, TextureAllocationFla
   MOZ_ASSERT(mFormat != gfx::SurfaceFormat::YUV, "This textureClient cannot use YCbCr data");
   MOZ_ASSERT(aSize.width > 0 && aSize.height > 0);
 
-  int bufSize
-    = ImageDataSerializer::ComputeMinBufferSize(aSize, mFormat);
+  if (aSize.width <= 0 || aSize.height <= 0) {
+    gfxDebug() << "Asking for buffer of invalid size " << aSize.width << "x" << aSize.height;
+    return false;
+  }
+
+  uint32_t bufSize = ImageDataSerializer::ComputeMinBufferSize(aSize, mFormat);
   if (!Allocate(bufSize)) {
     return false;
   }

gfx/layers/ipc/ISurfaceAllocator.cpp

@@ -111,6 +111,9 @@ ISurfaceAllocator::AllocSurfaceDescriptorWithCaps(const gfx::IntSize& aSize,
   gfx::SurfaceFormat format =
     gfxPlatform::GetPlatform()->Optimal2DFormatForContent(aContent);
   size_t size = ImageDataSerializer::ComputeMinBufferSize(aSize, format);
+  if (!size) {
+    return false;
+  }
   if (IsSameProcess()) {
     uint8_t *data = new (std::nothrow) uint8_t[size];
     if (!data) {

gfx/layers/ipc/SharedBufferManagerChild.cpp

@@ -8,6 +8,7 @@
 #include "base/task.h"                  // for NewRunnableFunction, etc
 #include "base/thread.h"                // for Thread
 #include "base/tracked.h"               // for FROM_HERE
+#include "mozilla/gfx/Logging.h"        // for gfxDebug
 #include "mozilla/layers/SharedBufferManagerChild.h"
 #include "mozilla/layers/SharedBufferManagerParent.h"
 #include "mozilla/StaticPtr.h"          // for StaticRefPtr
@@ -239,6 +240,11 @@ SharedBufferManagerChild::AllocGrallocBuffer(const gfx::IntSize& aSize,
                                              const uint32_t& aUsage,
                                              mozilla::layers::MaybeMagicGrallocBufferHandle* aBuffer)
 {
+  if (aSize.width <= 0 || aSize.height <= 0) {
+    gfxDebug() << "Asking for gralloc of invalid size " << aSize.width << "x" << aSize.height;
+    return false;
+  }
+
   if (InSharedBufferManagerChildThread()) {
     return SharedBufferManagerChild::AllocGrallocBufferNow(aSize, aFormat, aUsage, aBuffer);
   }
@@ -264,6 +270,9 @@ SharedBufferManagerChild::AllocGrallocBufferNow(const IntSize& aSize,
                                                 const uint32_t& aUsage,
                                                 mozilla::layers::MaybeMagicGrallocBufferHandle* aHandle)
 {
+  // These are protected functions, we can just assert and ask the caller to test
+  MOZ_ASSERT(aSize.width >= 0 && aSize.height >= 0);
+
 #ifdef MOZ_HAVE_SURFACEDESCRIPTORGRALLOC
   mozilla::layers::MaybeMagicGrallocBufferHandle handle;
   SendAllocateGrallocBuffer(aSize, aFormat, aUsage, &handle);

gfx/layers/ipc/SharedPlanarYCbCrImage.cpp

@@ -106,6 +106,9 @@ SharedPlanarYCbCrImage::AllocateAndGetNewBuffer(uint32_t aSize)
 {
   NS_ABORT_IF_FALSE(!mTextureClient, "This image already has allocated data");
   size_t size = YCbCrImageDataSerializer::ComputeMinBufferSize(aSize);
+  if (!size) {
+    return nullptr;
+  }
 
   mTextureClient = TextureClient::CreateWithBufferSize(mCompositable->GetForwarder(),
                                                        gfx::SurfaceFormat::YUV, size,
@@ -219,7 +222,7 @@ SharedPlanarYCbCrImage::Allocate(PlanarYCbCrData& aData)
                                                                mData.mCbCrSize);
   mSize = mData.mPicSize;
 
-  return true;
+  return mBufferSize > 0;
 }
 
 } // namespace