Bug 1463244 - cleanup of swizzle stride calculations. r=rhunt

MozReview-Commit-ID: GMXRKnu8zHB

gfx/2d/DataSurfaceHelpers.cpp

@@ -157,9 +157,16 @@ SurfaceToPackedBGRA(DataSourceSurface *aSurface)
   }
 
   IntSize size = aSurface->GetSize();
-
-  UniquePtr<uint8_t[]> imageBuffer(
-    new (std::nothrow) uint8_t[size.width * size.height * sizeof(uint32_t)]);
+  if (size.width < 0 || size.width >= INT32_MAX / 4) {
+    return nullptr;
+  }
+  int32_t stride = size.width * 4;
+  CheckedInt<size_t> bufferSize =
+    CheckedInt<size_t>(stride) * CheckedInt<size_t>(size.height);
+  if (!bufferSize.isValid()) {
+    return nullptr;
+  }
+  UniquePtr<uint8_t[]> imageBuffer(new (std::nothrow) uint8_t[bufferSize.value()]);
   if (!imageBuffer) {
     return nullptr;
   }
@@ -170,14 +177,14 @@ SurfaceToPackedBGRA(DataSourceSurface *aSurface)
   }
 
   CopySurfaceDataToPackedArray(map.mData, imageBuffer.get(), size,
-                               map.mStride, 4 * sizeof(uint8_t));
+                               map.mStride, 4);
 
   aSurface->Unmap();
 
   if (format == SurfaceFormat::B8G8R8X8) {
     // Convert BGRX to BGRA by setting a to 255.
-    SwizzleData(imageBuffer.get(), size.width * sizeof(uint32_t), SurfaceFormat::X8R8G8B8_UINT32,
-                imageBuffer.get(), size.width * sizeof(uint32_t), SurfaceFormat::A8R8G8B8_UINT32,
+    SwizzleData(imageBuffer.get(), stride, SurfaceFormat::X8R8G8B8_UINT32,
+                imageBuffer.get(), stride, SurfaceFormat::A8R8G8B8_UINT32,
                 size);
   }
 
@@ -196,8 +203,16 @@ SurfaceToPackedBGR(DataSourceSurface *aSurface)
   }
 
   IntSize size = aSurface->GetSize();
-
-  uint8_t* imageBuffer = new (std::nothrow) uint8_t[size.width * size.height * 3 * sizeof(uint8_t)];
+  if (size.width < 0 || size.width >= INT32_MAX / 3) {
+    return nullptr;
+  }
+  int32_t stride = size.width * 3;
+  CheckedInt<size_t> bufferSize =
+    CheckedInt<size_t>(stride) * CheckedInt<size_t>(size.height);
+  if (!bufferSize.isValid()) {
+    return nullptr;
+  }
+  uint8_t* imageBuffer = new (std::nothrow) uint8_t[bufferSize.value()];
   if (!imageBuffer) {
     return nullptr;
   }
@@ -209,7 +224,7 @@ SurfaceToPackedBGR(DataSourceSurface *aSurface)
   }
 
   SwizzleData(map.mData, map.mStride, SurfaceFormat::B8G8R8X8,
-              imageBuffer, size.width * 3, SurfaceFormat::B8G8R8,
+              imageBuffer, stride, SurfaceFormat::B8G8R8,
               size);
 
   aSurface->Unmap();

gfx/2d/Swizzle.cpp

@@ -259,7 +259,8 @@ static inline IntSize
 CollapseSize(const IntSize& aSize, int32_t aSrcStride, int32_t aDstStride)
 {
   if (aSrcStride == aDstStride &&
-      aSrcStride == 4 * aSize.width) {
+      (aSrcStride & 3) == 0 &&
+      aSrcStride / 4 == aSize.width) {
     CheckedInt32 area = CheckedInt32(aSize.width) * CheckedInt32(aSize.height);
     if (area.isValid()) {
       return IntSize(area.value(), 1);
@@ -268,6 +269,16 @@ CollapseSize(const IntSize& aSize, int32_t aSrcStride, int32_t aDstStride)
   return aSize;
 }
 
+static inline int32_t
+GetStrideGap(int32_t aWidth, SurfaceFormat aFormat, int32_t aStride)
+{
+  CheckedInt32 used = CheckedInt32(aWidth) * BytesPerPixel(aFormat);
+  if (!used.isValid() || used.value() < 0) {
+    return -1;
+  }
+  return aStride - used.value();
+}
+
 bool
 PremultiplyData(const uint8_t* aSrc, int32_t aSrcStride, SurfaceFormat aSrcFormat,
                 uint8_t* aDst, int32_t aDstStride, SurfaceFormat aDstFormat,
@@ -278,9 +289,12 @@ PremultiplyData(const uint8_t* aSrc, int32_t aSrcStride, SurfaceFormat aSrcForma
   }
   IntSize size = CollapseSize(aSize, aSrcStride, aDstStride);
   // Find gap from end of row to the start of the next row.
-  int32_t srcGap = aSrcStride - BytesPerPixel(aSrcFormat) * aSize.width;
-  int32_t dstGap = aDstStride - BytesPerPixel(aDstFormat) * aSize.width;
+  int32_t srcGap = GetStrideGap(aSize.width, aSrcFormat, aSrcStride);
+  int32_t dstGap = GetStrideGap(aSize.width, aDstFormat, aDstStride);
   MOZ_ASSERT(srcGap >= 0 && dstGap >= 0);
+  if (srcGap < 0 || dstGap < 0) {
+    return false;
+  }
 
 #define FORMAT_CASE_CALL(...) __VA_ARGS__(aSrc, srcGap, aDst, dstGap, size)
 
@@ -404,9 +418,12 @@ UnpremultiplyData(const uint8_t* aSrc, int32_t aSrcStride, SurfaceFormat aSrcFor
   }
   IntSize size = CollapseSize(aSize, aSrcStride, aDstStride);
   // Find gap from end of row to the start of the next row.
-  int32_t srcGap = aSrcStride - BytesPerPixel(aSrcFormat) * aSize.width;
-  int32_t dstGap = aDstStride - BytesPerPixel(aDstFormat) * aSize.width;
+  int32_t srcGap = GetStrideGap(aSize.width, aSrcFormat, aSrcStride);
+  int32_t dstGap = GetStrideGap(aSize.width, aDstFormat, aDstStride);
   MOZ_ASSERT(srcGap >= 0 && dstGap >= 0);
+  if (srcGap < 0 || dstGap < 0) {
+    return false;
+  }
 
 #define FORMAT_CASE_CALL(...) __VA_ARGS__(aSrc, srcGap, aDst, dstGap, size)
 
@@ -702,9 +719,12 @@ SwizzleData(const uint8_t* aSrc, int32_t aSrcStride, SurfaceFormat aSrcFormat,
   }
   IntSize size = CollapseSize(aSize, aSrcStride, aDstStride);
   // Find gap from end of row to the start of the next row.
-  int32_t srcGap = aSrcStride - BytesPerPixel(aSrcFormat) * aSize.width;
-  int32_t dstGap = aDstStride - BytesPerPixel(aDstFormat) * aSize.width;
+  int32_t srcGap = GetStrideGap(aSize.width, aSrcFormat, aSrcStride);
+  int32_t dstGap = GetStrideGap(aSize.width, aDstFormat, aDstStride);
   MOZ_ASSERT(srcGap >= 0 && dstGap >= 0);
+  if (srcGap < 0 || dstGap < 0) {
+    return false;
+  }
 
 #define FORMAT_CASE_CALL(...) __VA_ARGS__(aSrc, srcGap, aDst, dstGap, size)