Bug 1155252 - Don't allocate X11TextureClients bigger than xlib's maximum surface size. r=jrmuizel

dom/canvas/crashtests/crashtests.list

@@ -14,7 +14,7 @@ load 746813-1.html
 load 743499-negative-size.html
 skip-if(Android) load 767337-1.html
 skip-if(Android||B2G) load 780392-1.html # bug 833371 for B2G
-skip-if(Android||B2G) skip-if(gtk2Widget&&isDebugBuild) load 789933-1.html # bug 833371 for B2G, bug 1155252 for linux
+skip-if(Android||B2G) load 789933-1.html # bug 833371
 load 794463-1.html
 load 802926-1.html
 load 896047-1.html

gfx/layers/basic/TextureClientX11.cpp

@@ -112,7 +112,9 @@ TextureClientX11::AllocateForSurface(IntSize aSize, TextureAllocationFlags aText
   //MOZ_ASSERT(mFormat != gfx::FORMAT_YUV, "This TextureClient cannot use YCbCr data");
 
   MOZ_ASSERT(aSize.width >= 0 && aSize.height >= 0);
-  if (aSize.width <= 0 || aSize.height <= 0) {
+  if (aSize.width <= 0 || aSize.height <= 0 ||
+      aSize.width > XLIB_IMAGE_SIDE_SIZE_LIMIT ||
+      aSize.height > XLIB_IMAGE_SIDE_SIZE_LIMIT) {
     gfxDebug() << "Asking for X11 surface of invalid size " << aSize.width << "x" << aSize.height;
     return false;
   }

gfx/thebes/gfxXlibSurface.cpp

@@ -21,11 +21,6 @@
 
 using namespace mozilla;
 
-// Although the dimension parameters in the xCreatePixmapReq wire protocol are
-// 16-bit unsigned integers, the server's CreatePixmap returns BadAlloc if
-// either dimension cannot be represented by a 16-bit *signed* integer.
-#define XLIB_IMAGE_SIDE_SIZE_LIMIT 0x7fff
-
 gfxXlibSurface::gfxXlibSurface(Display *dpy, Drawable drawable, Visual *visual)
     : mPixmapTaken(false), mDisplay(dpy), mDrawable(drawable)
 #if defined(GL_PROVIDER_GLX)

gfx/thebes/gfxXlibSurface.h

@@ -17,6 +17,12 @@
 
 #include "nsSize.h"
 
+// Although the dimension parameters in the xCreatePixmapReq wire protocol are
+// 16-bit unsigned integers, the server's CreatePixmap returns BadAlloc if
+// either dimension cannot be represented by a 16-bit *signed* integer.
+#define XLIB_IMAGE_SIDE_SIZE_LIMIT 0x7fff
+
+
 class gfxXlibSurface final : public gfxASurface {
 public:
     // construct a wrapper around the specified drawable with dpy/visual.