Bug 572571 - Abort Core Animation rendering for windows that are too large. r=josh

gfx/thebes/nsCoreAnimationSupport.h

@@ -58,7 +58,8 @@ class THEBES_API nsCARenderer {
 public:
   nsCARenderer() : mCARenderer(nsnull), mPixelBuffer(nsnull), mOpenGLContext(nsnull),
                    mCGImage(nsnull), mCGData(nsnull), mIOSurface(nsnull), mFBO(nsnull),
-                   mIOTexture(nsnull) {}
+                   mIOTexture(nsnull), 
+                   mUnsupportedWidth(UINT32_MAX), mUnsupportedHeight(UINT32_MAX) {}
   ~nsCARenderer();
   nsresult SetupRenderer(void* aCALayer, int aWidth, int aHeight);
   nsresult Render(int aWidth, int aHeight, CGImageRef *aOutCAImage);
@@ -85,6 +86,8 @@ private:
   nsIOSurface       *mIOSurface;
   uint32_t           mFBO;
   uint32_t           mIOTexture;
+  uint32_t           mUnsupportedWidth;
+  uint32_t           mUnsupportedHeight;
 };
 
 typedef uint32_t IOSurfaceID;

gfx/thebes/nsCoreAnimationSupport.mm

@@ -449,6 +449,11 @@ nsresult nsCARenderer::SetupRenderer(void *aCALayer, int aWidth, int aHeight) {
   if (aWidth == 0 || aHeight == 0)
     return NS_ERROR_FAILURE;
 
+  if (aWidth == mUnsupportedWidth && 
+      aHeight == mUnsupportedHeight) {
+    return NS_ERROR_FAILURE;
+  }
+
   CALayer* layer = (CALayer*)aCALayer;
   CARenderer* caRenderer = nsnull;
 
@@ -464,6 +469,8 @@ nsresult nsCARenderer::SetupRenderer(void *aCALayer, int aWidth, int aHeight) {
     CGLError result = ::CGLCreatePBuffer(aWidth, aHeight,
                          GL_TEXTURE_2D, GL_RGBA, 0, &mPixelBuffer);
     if (result != kCGLNoError) {
+      mUnsupportedWidth = aWidth;
+      mUnsupportedHeight = aHeight;
       Destroy();
       return NS_ERROR_FAILURE;
     }
@@ -472,11 +479,15 @@ nsresult nsCARenderer::SetupRenderer(void *aCALayer, int aWidth, int aHeight) {
   GLint screen;
   CGLPixelFormatObj format;
   if (::CGLChoosePixelFormat(attributes, &format, &screen) != kCGLNoError) {
+    mUnsupportedWidth = aWidth;
+    mUnsupportedHeight = aHeight;
     Destroy();
     return NS_ERROR_FAILURE;
   }
 
   if (::CGLCreateContext(format, nsnull, &mOpenGLContext) != kCGLNoError) {
+    mUnsupportedWidth = aWidth;
+    mUnsupportedHeight = aHeight;
     Destroy();
     return NS_ERROR_FAILURE;
   }
@@ -486,6 +497,8 @@ nsresult nsCARenderer::SetupRenderer(void *aCALayer, int aWidth, int aHeight) {
                             options:nil] retain];
   mCARenderer = caRenderer;
   if (caRenderer == nil) {
+    mUnsupportedWidth = aWidth;
+    mUnsupportedHeight = aHeight;
     Destroy();
     return NS_ERROR_FAILURE;
   }
@@ -498,6 +511,8 @@ nsresult nsCARenderer::SetupRenderer(void *aCALayer, int aWidth, int aHeight) {
   if (!mIOSurface) {
     mCGData = malloc(aWidth*aHeight*4);
     if (!mCGData) {
+      mUnsupportedWidth = aWidth;
+      mUnsupportedHeight = aHeight;
       Destroy();
     }
     memset(mCGData, 0, aWidth*aHeight*4);
@@ -508,6 +523,8 @@ nsresult nsCARenderer::SetupRenderer(void *aCALayer, int aWidth, int aHeight) {
                                         cgdata_release_callback);
     if (!dataProvider) {
       cgdata_release_callback(mCGData, mCGData, aHeight*aWidth*4);
+      mUnsupportedWidth = aWidth;
+      mUnsupportedHeight = aHeight;
       Destroy();
       return NS_ERROR_FAILURE;
     }
@@ -523,6 +540,8 @@ nsresult nsCARenderer::SetupRenderer(void *aCALayer, int aWidth, int aHeight) {
       ::CGColorSpaceRelease(colorSpace);
     }
     if (!mCGImage) {
+      mUnsupportedWidth = aWidth;
+      mUnsupportedHeight = aHeight;
       Destroy();
       return NS_ERROR_FAILURE;
     }
@@ -554,6 +573,8 @@ nsresult nsCARenderer::SetupRenderer(void *aCALayer, int aWidth, int aHeight) {
       NS_ERROR("FBO not supported");
       if (oldContext)
         ::CGLSetCurrentContext(oldContext);
+      mUnsupportedWidth = aWidth;
+      mUnsupportedHeight = aHeight;
       Destroy();
       return NS_ERROR_FAILURE; 
     }
@@ -577,6 +598,8 @@ nsresult nsCARenderer::SetupRenderer(void *aCALayer, int aWidth, int aHeight) {
   GLenum result = ::glGetError();
   if (result != GL_NO_ERROR) {
     NS_ERROR("Unexpected OpenGL Error");
+    mUnsupportedWidth = aWidth;
+    mUnsupportedHeight = aHeight;
     Destroy();
     if (oldContext)
       ::CGLSetCurrentContext(oldContext);
@@ -635,6 +658,7 @@ nsresult nsCARenderer::Render(int aWidth, int aHeight,
     if (SetupRenderer(caLayer, aWidth, aHeight) != NS_OK) {
       return NS_ERROR_FAILURE;
     }
+
     caRenderer = (CARenderer*)mCARenderer;
   }