Bug 1574592 - When rendering into a copy target with CompositorOGL, and the window uses native layers for rendering, render into an offscreen RenderTarget instead of using a native layer. r=mattwoodrow

On platforms that don't use native layers for rendering, we usually want reftests
to see exactly what's going to end up in the window. It's conceivable that
rendering into offscreen framebuffers might miss bugs on those platforms (though
I'm not sure if we've seen any evidence of this).
But when we're rendering to native layers, we have non-default framebuffers either
way, so we might as well create our own framebuffer rather than asking a native
layer for one. We're not interested in getting this rendering to the screen, so
it's better to leave the native layer out of this.

Differential Revision: https://phabricator.services.mozilla.com/D42406

--HG--
extra : moz-landing-system : lando

gfx/layers/opengl/CompositorOGL.cpp

@@ -206,6 +206,11 @@ CompositorOGL::CompositorOGL(CompositorBridgeParent* aParent,
       mDestroyed(false),
       mViewportSize(0, 0),
       mCurrentProgram(nullptr) {
+  if (aWidget->GetNativeLayerRoot()) {
+    // We can only render into native layers, our GLContext won't have a usable
+    // default framebuffer.
+    mCanRenderToDefaultFramebuffer = false;
+  }
 #ifdef XP_DARWIN
   TextureSync::RegisterTextureSourceProvider(this);
 #endif
@@ -898,12 +903,24 @@ void CompositorOGL::BeginFrame(const nsIntRegion& aInvalidRegion,
                                  LOCAL_GL_ONE, LOCAL_GL_ONE_MINUS_SRC_ALPHA);
   mGLContext->fEnable(LOCAL_GL_BLEND);
 
-  RefPtr<CompositingRenderTargetOGL> rt =
-      aNativeLayer ? RenderTargetForNativeLayer(aNativeLayer)
-                   : CompositingRenderTargetOGL::RenderTargetForWindow(
-                         this, IntSize(width, height));
+  RefPtr<CompositingRenderTarget> rt;
+  if (mTarget) {
+    if (mCanRenderToDefaultFramebuffer) {
+      rt = CompositingRenderTargetOGL::RenderTargetForWindow(this, rect.Size());
+    } else {
+      rt = CreateRenderTarget(rect, INIT_MODE_CLEAR);
+    }
+  } else if (aNativeLayer) {
+    rt = RenderTargetForNativeLayer(aNativeLayer);
+    mCurrentNativeLayer = aNativeLayer;
+  } else {
+    MOZ_RELEASE_ASSERT(mCanRenderToDefaultFramebuffer);
+    rt = CompositingRenderTargetOGL::RenderTargetForWindow(this, rect.Size());
+  }
+
   if (!rt) {
     *aRenderBoundsOut = IntRect();
+    mCurrentNativeLayer = nullptr;
     return;
   }
 
@@ -912,7 +929,6 @@ void CompositorOGL::BeginFrame(const nsIntRegion& aInvalidRegion,
 
   SetRenderTarget(rt);
   mWindowRenderTarget = mCurrentRenderTarget;
-  mCurrentNativeLayer = aNativeLayer;
 
   if (aClipRectOut && !aClipRectIn) {
     aClipRectOut->SetRect(0, 0, width, height);

gfx/layers/opengl/CompositorOGL.h

@@ -302,6 +302,7 @@ class CompositorOGL final : public Compositor {
   RefPtr<GLContext> mGLContext;
   UniquePtr<GLBlitTextureImageHelper> mBlitTextureImageHelper;
   gfx::Matrix4x4 mProjMatrix;
+  bool mCanRenderToDefaultFramebuffer = true;
 
 #ifdef XP_DARWIN
   nsTArray<RefPtr<BufferTextureHost>> mMaybeUnlockBeforeNextComposition;