Bug 539356 - Part 8a - Add END_NO_COMPOSITE to EndTransactionFlags and implement in for all LayerManagers. r=roc

gfx/layers/Layers.h

@@ -225,7 +225,8 @@ public:
 
   enum EndTransactionFlags {
     END_DEFAULT = 0,
-    END_NO_IMMEDIATE_REDRAW = 1 << 0  // Do not perform the drawing phase
+    END_NO_IMMEDIATE_REDRAW = 1 << 0,  // Do not perform the drawing phase
+    END_NO_COMPOSITE = 1 << 1 // Do not composite after drawing thebes layer contents.
   };
 
   /**

gfx/layers/basic/BasicLayerManager.cpp

@@ -421,6 +421,12 @@ BasicLayerManager::EndTransactionInternal(DrawThebesLayerCallback aCallback,
 
   mTransactionIncomplete = false;
 
+  if (aFlags & END_NO_COMPOSITE) {
+    // TODO: We should really just set mTarget to null and make sure we can handle that further down the call chain
+    nsRefPtr<gfxASurface> surf = gfxPlatform::GetPlatform()->CreateOffscreenSurface(gfxIntSize(1, 1), gfxASurface::CONTENT_COLOR);
+    mTarget = new gfxContext(surf);
+  }
+
   if (mTarget && mRoot && !(aFlags & END_NO_IMMEDIATE_REDRAW)) {
     nsIntRect clipRect;
     if (HasShadowManager()) {
@@ -450,9 +456,19 @@ BasicLayerManager::EndTransactionInternal(DrawThebesLayerCallback aCallback,
       }
     }
 
-    PaintLayer(mTarget, mRoot, aCallback, aCallbackData, nullptr);
-    if (mWidget) {
-      FlashWidgetUpdateArea(mTarget);
+    if (aFlags & END_NO_COMPOSITE) {
+      if (IsRetained()) {
+        // Clip the destination out so that we don't draw to it, and
+        // only end up validating ThebesLayers.
+        mTarget->Clip(gfxRect(0, 0, 0, 0));
+        PaintLayer(mTarget, mRoot, aCallback, aCallbackData, nullptr);
+      }
+      // If we're not retained, then don't composite means do nothing at all.
+    } else {
+      PaintLayer(mTarget, mRoot, aCallback, aCallbackData, nullptr);
+      if (mWidget) {
+        FlashWidgetUpdateArea(mTarget);
+      }
     }
 
     if (!mTransactionIncomplete) {

gfx/layers/basic/BasicThebesLayer.cpp

@@ -27,9 +27,7 @@ BasicThebesLayer::CreateBuffer(Buffer::ContentType aType, const nsIntSize& aSize
       referenceSurface = defaultTarget->CurrentSurface();
     } else {
       nsIWidget* widget = BasicManager()->GetRetainerWidget();
-      if (widget) {
-        referenceSurface = widget->GetThebesSurface();
-      } else {
+      if (!widget || !(referenceSurface = widget->GetThebesSurface())) {
         referenceSurface = BasicManager()->GetTarget()->CurrentSurface();
       }
     }

gfx/layers/d3d10/LayerManagerD3D10.cpp

@@ -362,7 +362,7 @@ LayerManagerD3D10::EndTransaction(DrawThebesLayerCallback aCallback,
     Log();
 #endif
 
-    Render();
+    Render(aFlags);
     mCurrentCallbackInfo.Callback = nullptr;
     mCurrentCallbackInfo.CallbackData = nullptr;
   }
@@ -703,10 +703,14 @@ LayerManagerD3D10::EnsureReadbackManager()
 }
 
 void
-LayerManagerD3D10::Render()
+LayerManagerD3D10::Render(EndTransactionFlags aFlags)
 {
   static_cast<LayerD3D10*>(mRoot->ImplData())->Validate();
 
+  if (aFlags & END_NO_COMPOSITE) {
+    return;
+  }
+
   SetupPipeline();
 
   float black[] = { 0, 0, 0, 0 };

gfx/layers/d3d10/LayerManagerD3D10.h

@@ -178,7 +178,7 @@ private:
   void VerifyBufferSize();
   void EnsureReadbackManager();
 
-  void Render();
+  void Render(EndTransactionFlags aFlags);
 
   nsRefPtr<ID3D10Device1> mDevice;
 

gfx/layers/d3d9/CanvasLayerD3D9.cpp

@@ -187,6 +187,9 @@ void
 CanvasLayerD3D9::RenderLayer()
 {
   UpdateSurface();
+  if (mD3DManager->CompositingDisabled()) {
+    return;
+  }
   FireDidTransactionCallback();
 
   if (!mTexture)
@@ -355,7 +358,7 @@ ShadowCanvasLayerD3D9::GetLayer()
 void
 ShadowCanvasLayerD3D9::RenderLayer()
 {
-  if (!mBuffer) {
+  if (!mBuffer || mD3DManager->CompositingDisabled()) {
     return;
   }
 

gfx/layers/d3d9/ColorLayerD3D9.cpp

@@ -19,6 +19,9 @@ RenderColorLayerD3D9(ColorLayer* aLayer, LayerManagerD3D9 *aManager)
 {
   // XXX we might be able to improve performance by using
   // IDirect3DDevice9::Clear
+  if (aManager->CompositingDisabled()) {
+    return;
+  }
 
   nsIntRect visibleRect = aLayer->GetEffectiveVisibleRegion().GetBounds();
 

gfx/layers/d3d9/ContainerLayerD3D9.cpp

@@ -147,20 +147,23 @@ ContainerRender(Container* aContainer,
 
   aContainer->mSupportsComponentAlphaChildren = false;
   if (useIntermediate) {
-    aManager->device()->GetRenderTarget(0, getter_AddRefs(previousRenderTarget));
-    HRESULT hr = aManager->device()->CreateTexture(visibleRect.width, visibleRect.height, 1,
-                                                   D3DUSAGE_RENDERTARGET, D3DFMT_A8R8G8B8,
-                                                   D3DPOOL_DEFAULT, getter_AddRefs(renderTexture),
-                                                   NULL);
-    if (FAILED(hr)) {
-      aManager->ReportFailure(NS_LITERAL_CSTRING("ContainerLayerD3D9::ContainerRender(): Failed to create texture"),
-                              hr);
-      return;
-    }
-
     nsRefPtr<IDirect3DSurface9> renderSurface;
-    renderTexture->GetSurfaceLevel(0, getter_AddRefs(renderSurface));
-    aManager->device()->SetRenderTarget(0, renderSurface);
+    if (!aManager->CompositingDisabled()) {
+      aManager->device()->GetRenderTarget(0, getter_AddRefs(previousRenderTarget));
+      HRESULT hr = aManager->device()->CreateTexture(visibleRect.width, visibleRect.height, 1,
+                                                     D3DUSAGE_RENDERTARGET, D3DFMT_A8R8G8B8,
+                                                     D3DPOOL_DEFAULT, getter_AddRefs(renderTexture),
+                                                     NULL);
+      if (FAILED(hr)) {
+        aManager->ReportFailure(NS_LITERAL_CSTRING("ContainerLayerD3D9::ContainerRender(): Failed to create texture"),
+                                hr);
+        return;
+      }
+
+      nsRefPtr<IDirect3DSurface9> renderSurface;
+      renderTexture->GetSurfaceLevel(0, getter_AddRefs(renderSurface));
+      aManager->device()->SetRenderTarget(0, renderSurface);
+    }
 
     if (aContainer->mVisibleRegion.GetNumRects() == 1 && 
         (aContainer->GetContentFlags() & aContainer->CONTENT_OPAQUE)) {
@@ -182,12 +185,14 @@ ContainerRender(Container* aContainer,
         ::OffsetRect(&src,
                      visibleRect.x + PRInt32(transform.x0),
                      visibleRect.y + PRInt32(transform.y0));
-        hr = aManager->device()->
-          StretchRect(previousRenderTarget, &src, renderSurface, &dest, D3DTEXF_NONE);
+        if (!aManager->CompositingDisabled()) {
+          hr = aManager->device()->
+            StretchRect(previousRenderTarget, &src, renderSurface, &dest, D3DTEXF_NONE);
+        }
       }
       if (hr == S_OK) {
         aContainer->mSupportsComponentAlphaChildren = true;
-      } else {
+      } else if (!aManager->CompositingDisabled()) {
         aManager->device()->
           Clear(0, 0, D3DCLEAR_TARGET, D3DCOLOR_RGBA(0, 0, 0, 0), 0, 0);
       }
@@ -256,7 +261,7 @@ ContainerRender(Container* aContainer,
     
   aManager->device()->SetScissorRect(&containerD3D9ClipRect);
 
-  if (useIntermediate) {
+  if (useIntermediate && !aManager->CompositingDisabled()) {
     aManager->device()->SetRenderTarget(0, previousRenderTarget);
     aManager->device()->SetVertexShaderConstantF(CBvRenderTargetOffset, previousRenderTargetOffset, 1);
     aManager->device()->SetVertexShaderConstantF(CBmProjection, &oldViewMatrix[0][0], 4);

gfx/layers/d3d9/ImageLayerD3D9.cpp

@@ -355,7 +355,7 @@ void
 ImageLayerD3D9::RenderLayer()
 {
   ImageContainer *container = GetContainer();
-  if (!container) {
+  if (!container || mD3DManager->CompositingDisabled()) {
     return;
   }
 
@@ -609,6 +609,10 @@ ShadowImageLayerD3D9::GetLayer()
 void
 ShadowImageLayerD3D9::RenderLayer()
 {
+  if (mD3DManager->CompositingDisabled()) {
+    return;
+  }
+
   if (mBuffer) {
     mBuffer->RenderTo(mD3DManager, GetEffectiveVisibleRegion());
   } else if (mYCbCrImage) {

gfx/layers/d3d9/LayerManagerD3D9.cpp

@@ -149,6 +149,7 @@ LayerManagerD3D9::EndTransaction(DrawThebesLayerCallback aCallback,
     // so we don't need to pass any global transform here.
     mRoot->ComputeEffectiveTransforms(gfx3DMatrix());
 
+    SetCompositingDisabled(aFlags & END_NO_COMPOSITE);
     Render();
     /* Clean this out for sanity */
     mCurrentCallbackInfo.Callback = NULL;
@@ -284,6 +285,12 @@ LayerManagerD3D9::Render()
   deviceManager()->SetupRenderState();
 
   SetupPipeline();
+
+  if (CompositingDisabled()) {
+    static_cast<LayerD3D9*>(mRoot->ImplData())->RenderLayer();
+    return;
+  }
+
   nsIntRect rect;
   mWidget->GetClientBounds(rect);
 

gfx/layers/d3d9/LayerManagerD3D9.h

@@ -176,6 +176,9 @@ public:
 
   void ReportFailure(const nsACString &aMsg, HRESULT aCode);
 
+  bool CompositingDisabled() { return mCompositingDisabled; }
+  void SetCompositingDisabled(bool aCompositingDisabled) { mCompositingDisabled = aCompositingDisabled; }
+
 private:
   /* Default device manager instance */
   static DeviceManagerD3D9 *mDefaultDeviceManager;
@@ -208,6 +211,12 @@ private:
    */
   PRUint32 mDeviceResetCount;
 
+  /*
+   * True if we should only be drawing layer contents, not
+   * compositing them to the target.
+   */
+  bool mCompositingDisabled;
+
   /*
    * Render the current layer tree to the active target.
    */

gfx/layers/d3d9/ThebesLayerD3D9.cpp

@@ -237,6 +237,10 @@ ThebesLayerD3D9::RenderThebesLayer(ReadbackProcessor* aReadback)
     mValidRegion = neededRegion;
   }
 
+  if (mD3DManager->CompositingDisabled()) {
+    return;
+  }
+
   SetShaderTransformAndOpacity();
 
   if (mode == SURFACE_COMPONENT_ALPHA) {
@@ -655,7 +659,7 @@ ShadowThebesLayerD3D9::IsEmpty()
 void
 ShadowThebesLayerD3D9::RenderThebesLayer()
 {
-  if (!mBuffer) {
+  if (!mBuffer || mD3DManager->CompositingDisabled()) {
     return;
   }
   NS_ABORT_IF_FALSE(mBuffer, "should have a buffer here");

gfx/layers/opengl/CanvasLayerOGL.cpp

@@ -229,6 +229,9 @@ CanvasLayerOGL::RenderLayer(int aPreviousDestination,
                             const nsIntPoint& aOffset)
 {
   UpdateSurface();
+  if (mOGLManager->CompositingDisabled()) {
+    return;
+  }
   FireDidTransactionCallback();
 
   mOGLManager->MakeCurrent();
@@ -446,6 +449,9 @@ ShadowCanvasLayerOGL::RenderLayer(int aPreviousFrameBuffer,
     return;
   }
 
+  if (mOGLManager->CompositingDisabled()) {
+    return;
+  }
   mOGLManager->MakeCurrent();
 
   gfx3DMatrix effectiveTransform = GetEffectiveTransform();

gfx/layers/opengl/ColorLayerOGL.cpp

@@ -12,6 +12,10 @@ static void
 RenderColorLayer(ColorLayer* aLayer, LayerManagerOGL *aManager,
                  const nsIntPoint& aOffset)
 {
+  if (aManager->CompositingDisabled()) {
+    return;
+  }
+
   aManager->MakeCurrent();
 
   // XXX we might be able to improve performance by using glClear

gfx/layers/opengl/ContainerLayerOGL.cpp

@@ -179,12 +179,14 @@ ContainerRender(Container* aContainer,
     }
 
     aContainer->gl()->PushViewportRect();
-    framebufferRect -= childOffset; 
-    aManager->CreateFBOWithTexture(framebufferRect,
-                                   mode,
-                                   aPreviousFrameBuffer,
-                                   &frameBuffer,
-                                   &containerSurface);
+    framebufferRect -= childOffset;
+    if (!aManager->CompositingDisabled()) {
+      aManager->CreateFBOWithTexture(framebufferRect,
+                                     mode,
+                                     aPreviousFrameBuffer,
+                                     &frameBuffer,
+                                     &containerSurface);
+    }
     childOffset.x = visibleRect.x;
     childOffset.y = visibleRect.y;
   } else {
@@ -239,45 +241,47 @@ ContainerRender(Container* aContainer,
     aManager->SetupPipeline(viewport.width, viewport.height,
                             LayerManagerOGL::ApplyWorldTransform);
     aContainer->gl()->PopScissorRect();
-
     aContainer->gl()->fBindFramebuffer(LOCAL_GL_FRAMEBUFFER, aPreviousFrameBuffer);
-    aContainer->gl()->fDeleteFramebuffers(1, &frameBuffer);
 
-    aContainer->gl()->fActiveTexture(LOCAL_GL_TEXTURE0);
+    if (!aManager->CompositingDisabled()) {
+      aContainer->gl()->fDeleteFramebuffers(1, &frameBuffer);
 
-    aContainer->gl()->fBindTexture(aManager->FBOTextureTarget(), containerSurface);
+      aContainer->gl()->fActiveTexture(LOCAL_GL_TEXTURE0);
 
-    MaskType maskType = MaskNone;
-    if (aContainer->GetMaskLayer()) {
-      if (!aContainer->GetTransform().CanDraw2D()) {
-        maskType = Mask3d;
-      } else {
-        maskType = Mask2d;
+      aContainer->gl()->fBindTexture(aManager->FBOTextureTarget(), containerSurface);
+
+      MaskType maskType = MaskNone;
+      if (aContainer->GetMaskLayer()) {
+        if (!aContainer->GetTransform().CanDraw2D()) {
+          maskType = Mask3d;
+        } else {
+          maskType = Mask2d;
+        }
       }
+      ShaderProgramOGL *rgb =
+        aManager->GetFBOLayerProgram(maskType);
+
+      rgb->Activate();
+      rgb->SetLayerQuadRect(visibleRect);
+      rgb->SetLayerTransform(transform);
+      rgb->SetLayerOpacity(opacity);
+      rgb->SetRenderOffset(aOffset);
+      rgb->SetTextureUnit(0);
+      rgb->LoadMask(aContainer->GetMaskLayer());
+
+      if (rgb->GetTexCoordMultiplierUniformLocation() != -1) {
+        // 2DRect case, get the multiplier right for a sampler2DRect
+        rgb->SetTexCoordMultiplier(visibleRect.width, visibleRect.height);
+      }
+
+      // Drawing is always flipped, but when copying between surfaces we want to avoid
+      // this. Pass true for the flip parameter to introduce a second flip
+      // that cancels the other one out.
+      aManager->BindAndDrawQuad(rgb, true);
+
+      // Clean up resources.  This also unbinds the texture.
+      aContainer->gl()->fDeleteTextures(1, &containerSurface);
     }
-    ShaderProgramOGL *rgb =
-      aManager->GetFBOLayerProgram(maskType);
-
-    rgb->Activate();
-    rgb->SetLayerQuadRect(visibleRect);
-    rgb->SetLayerTransform(transform);
-    rgb->SetLayerOpacity(opacity);
-    rgb->SetRenderOffset(aOffset);
-    rgb->SetTextureUnit(0);
-    rgb->LoadMask(aContainer->GetMaskLayer());
-
-    if (rgb->GetTexCoordMultiplierUniformLocation() != -1) {
-      // 2DRect case, get the multiplier right for a sampler2DRect
-      rgb->SetTexCoordMultiplier(visibleRect.width, visibleRect.height);
-    }
-
-    // Drawing is always flipped, but when copying between surfaces we want to avoid
-    // this. Pass true for the flip parameter to introduce a second flip
-    // that cancels the other one out.
-    aManager->BindAndDrawQuad(rgb, true);
-
-    // Clean up resources.  This also unbinds the texture.
-    aContainer->gl()->fDeleteTextures(1, &containerSurface);
   } else {
     aContainer->gl()->PopScissorRect();
   }

gfx/layers/opengl/ImageLayerOGL.cpp

@@ -225,7 +225,7 @@ ImageLayerOGL::RenderLayer(int,
 {
   nsRefPtr<ImageContainer> container = GetContainer();
 
-  if (!container)
+  if (!container || mOGLManager->CompositingDisabled())
     return;
 
   mOGLManager->MakeCurrent();
@@ -875,6 +875,9 @@ void
 ShadowImageLayerOGL::RenderLayer(int aPreviousFrameBuffer,
                                  const nsIntPoint& aOffset)
 {
+  if (mOGLManager->CompositingDisabled()) {
+    return;
+  }
   mOGLManager->MakeCurrent();
   if (mImageContainerID) {
     ImageContainerParent::SetCompositorIDForImage(mImageContainerID,

gfx/layers/opengl/LayerManagerOGL.cpp

@@ -411,6 +411,7 @@ LayerManagerOGL::EndTransaction(DrawThebesLayerCallback aCallback,
 
     mThebesLayerCallback = aCallback;
     mThebesLayerCallbackData = aCallbackData;
+    SetCompositingDisabled(aFlags & END_NO_COMPOSITE);
 
     Render();
 
@@ -774,6 +775,13 @@ LayerManagerOGL::Render()
     mGLContext->fScissor(0, 0, width, height);
   }
 
+  if (CompositingDisabled()) {
+    RootLayer()->RenderLayer(mGLContext->IsDoubleBuffered() ? 0 : mBackBufferFBO,
+                             nsIntPoint(0, 0));
+    mGLContext->fBindBuffer(LOCAL_GL_ARRAY_BUFFER, 0);
+    return;
+  }
+
   mGLContext->fEnable(LOCAL_GL_SCISSOR_TEST);
 
   // If the Java compositor is being used, this clear will be done in

gfx/layers/opengl/LayerManagerOGL.h

@@ -350,6 +350,9 @@ public:
    */
   void SetSurfaceSize(int width, int height);
 
+  bool CompositingDisabled() { return mCompositingDisabled; }
+  void SetCompositingDisabled(bool aCompositingDisabled) { mCompositingDisabled = aCompositingDisabled; }
+
 private:
   /** Widget associated with this layer manager */
   nsIWidget *mWidget;
@@ -391,6 +394,7 @@ private:
 
   /** Misc */
   bool mHasBGRA;
+  bool mCompositingDisabled;
 
   /**
    * When rendering to an EGL surface (e.g. on Android), we rely on being told

gfx/layers/opengl/ThebesLayerOGL.cpp

@@ -88,6 +88,8 @@ public:
   void RenderTo(const nsIntPoint& aOffset, LayerManagerOGL* aManager,
                 PRUint32 aFlags);
 
+  void EndUpdate();
+
   nsIntSize GetSize() {
     if (mTexImage)
       return mTexImage->GetSize();
@@ -108,6 +110,17 @@ protected:
   bool mInitialised;
 };
 
+void ThebesLayerBufferOGL::EndUpdate()
+{
+  if (mTexImage && mTexImage->InUpdate()) {
+    mTexImage->EndUpdate();
+  }
+
+  if (mTexImageOnWhite && mTexImageOnWhite->InUpdate()) {
+    mTexImageOnWhite->EndUpdate();
+  }
+}
+
 void
 ThebesLayerBufferOGL::RenderTo(const nsIntPoint& aOffset,
                                LayerManagerOGL* aManager,
@@ -118,13 +131,7 @@ ThebesLayerBufferOGL::RenderTo(const nsIntPoint& aOffset,
   if (!mTexImage || !Initialised())
     return;
 
-  if (mTexImage->InUpdate()) {
-    mTexImage->EndUpdate();
-  }
-
-  if (mTexImageOnWhite && mTexImageOnWhite->InUpdate()) {
-    mTexImageOnWhite->EndUpdate();
-  }
+  EndUpdate();
 
 #ifdef MOZ_DUMP_PAINTING
   if (gfxUtils::sDumpPainting) {
@@ -838,6 +845,11 @@ ThebesLayerOGL::RenderLayer(int aPreviousFrameBuffer,
     }
   }
 
+  if (mOGLManager->CompositingDisabled()) {
+    mBuffer->EndUpdate();
+    return;
+  }
+
   // Drawing thebes layers can change the current context, reset it.
   gl()->MakeCurrent();
 
@@ -1099,7 +1111,7 @@ void
 ShadowThebesLayerOGL::RenderLayer(int aPreviousFrameBuffer,
                                   const nsIntPoint& aOffset)
 {
-  if (!mBuffer) {
+  if (!mBuffer || mOGLManager->CompositingDisabled()) {
     return;
   }
   NS_ABORT_IF_FALSE(mBuffer, "should have a buffer here");

widget/cocoa/nsChildView.mm

@@ -2556,6 +2556,9 @@ NSEvent* gLastDragMouseDownEvent = nil;
       [self setGLContext:glContext];
     }
 
+    [glContext setView:self];
+    [glContext update];
+
     mGeckoChild->DispatchWindowEvent(paintEvent);
 
     // Force OpenGL to refresh the very first time we draw. This works around a