Bug 1574592 - Create a render target per IOSurface, laying the groundwork for multiple native layers. r=mattwoodrow

This makes CompositorOGL always render into offscreen render targets.
CompositorOGL no longer expects the GLContext to have a default framebuffer.
This duplicates a bunch of code from GLContextCGL, but that code can be removed
from GLContextCGL once WebRender no longer needs it.
This also makes it so that we don't recreate render targets on every frame;
instead, we create one render target for every IOSurface in the native layer's
"swap chain" and cycle through those render targets. We only throw away the
render targets when the window is resized or closed.

Differential Revision: https://phabricator.services.mozilla.com/D42405

--HG--
extra : moz-landing-system : lando

gfx/layers/opengl/CompositorOGL.cpp

@@ -754,19 +754,62 @@ void CompositorOGL::ClearRect(const gfx::Rect& aRect) {
 }
 
 #ifdef XP_MACOSX
-class SurfaceRegistryWrapperAroundGLContextCGL
+class SurfaceRegistryWrapperAroundCompositorOGL
     : public layers::IOSurfaceRegistry {
  public:
-  explicit SurfaceRegistryWrapperAroundGLContextCGL(gl::GLContextCGL* aContext)
-      : mContext(aContext) {}
+  explicit SurfaceRegistryWrapperAroundCompositorOGL(CompositorOGL* aCompositor)
+      : mCompositor(aCompositor) {}
   void RegisterSurface(CFTypeRefPtr<IOSurfaceRef> aSurface) override {
-    mContext->RegisterIOSurface(aSurface.get());
+    mCompositor->RegisterIOSurface((IOSurfacePtr)aSurface.get());
   }
   void UnregisterSurface(CFTypeRefPtr<IOSurfaceRef> aSurface) override {
-    mContext->UnregisterIOSurface(aSurface.get());
+    mCompositor->UnregisterIOSurface((IOSurfacePtr)aSurface.get());
   }
-  RefPtr<gl::GLContextCGL> mContext;
+  RefPtr<CompositorOGL> mCompositor;
 };
+
+void CompositorOGL::RegisterIOSurface(IOSurfacePtr aSurface) {
+  MOZ_RELEASE_ASSERT(mRegisteredIOSurfaceRenderTargets.find(aSurface) ==
+                         mRegisteredIOSurfaceRenderTargets.end(),
+                     "double-registering IOSurface");
+
+  IOSurfaceRef surface = (IOSurfaceRef)aSurface;
+  auto glContextCGL = GLContextCGL::Cast(mGLContext);
+  MOZ_RELEASE_ASSERT(glContextCGL, "Unexpected GLContext type");
+
+  IntSize size(IOSurfaceGetWidth(surface), IOSurfaceGetHeight(surface));
+
+  mGLContext->MakeCurrent();
+  GLuint tex = mGLContext->CreateTexture();
+  {
+    const ScopedBindTexture bindTex(mGLContext, tex,
+                                    LOCAL_GL_TEXTURE_RECTANGLE_ARB);
+    CGLTexImageIOSurface2D(glContextCGL->GetCGLContext(),
+                           LOCAL_GL_TEXTURE_RECTANGLE_ARB, LOCAL_GL_RGBA,
+                           size.width, size.height, LOCAL_GL_BGRA,
+                           LOCAL_GL_UNSIGNED_INT_8_8_8_8_REV, surface, 0);
+  }
+
+  GLuint fbo = mGLContext->CreateFramebuffer();
+  {
+    const ScopedBindFramebuffer bindFB(mGLContext, fbo);
+    mGLContext->fFramebufferTexture2D(LOCAL_GL_FRAMEBUFFER,
+                                      LOCAL_GL_COLOR_ATTACHMENT0,
+                                      LOCAL_GL_TEXTURE_RECTANGLE_ARB, tex, 0);
+  }
+
+  RefPtr<CompositingRenderTargetOGL> rt =
+      new CompositingRenderTargetOGL(this, gfx::IntPoint(), tex, fbo);
+  rt->Initialize(size, size, LOCAL_GL_TEXTURE_RECTANGLE_ARB, INIT_MODE_NONE);
+
+  mRegisteredIOSurfaceRenderTargets.insert({aSurface, rt});
+}
+
+void CompositorOGL::UnregisterIOSurface(IOSurfacePtr aSurface) {
+  size_t removeCount = mRegisteredIOSurfaceRenderTargets.erase(aSurface);
+  MOZ_RELEASE_ASSERT(removeCount == 1,
+                     "Unregistering IOSurface that's not registered");
+}
 #endif
 
 already_AddRefed<CompositingRenderTargetOGL>
@@ -781,19 +824,21 @@ CompositorOGL::RenderTargetForNativeLayer(NativeLayer* aNativeLayer) {
       nativeLayer->GetSurfaceRegistry();
   if (!currentRegistry) {
     nativeLayer->SetSurfaceRegistry(
-        MakeAndAddRef<SurfaceRegistryWrapperAroundGLContextCGL>(glContextCGL));
+        MakeAndAddRef<SurfaceRegistryWrapperAroundCompositorOGL>(this));
   } else {
-    MOZ_RELEASE_ASSERT(static_cast<SurfaceRegistryWrapperAroundGLContextCGL*>(
+    MOZ_RELEASE_ASSERT(static_cast<SurfaceRegistryWrapperAroundCompositorOGL*>(
                            currentRegistry.get())
-                           ->mContext == glContextCGL);
+                           ->mCompositor == this);
   }
   CFTypeRefPtr<IOSurfaceRef> surf = nativeLayer->NextSurface();
   if (!surf) {
     return nullptr;
   }
-  glContextCGL->UseRegisteredIOSurfaceForDefaultFramebuffer(surf.get());
-  return CompositingRenderTargetOGL::RenderTargetForWindow(
-      this, nativeLayer->GetRect().Size());
+
+  auto match = mRegisteredIOSurfaceRenderTargets.find((IOSurfacePtr)surf.get());
+  MOZ_RELEASE_ASSERT(match != mRegisteredIOSurfaceRenderTargets.end(),
+                     "IOSurface has not been registered with this Compositor");
+  return do_AddRef(match->second);
 #else
   MOZ_CRASH("Unexpected native layer on this platform");
 #endif

gfx/layers/opengl/CompositorOGL.h

@@ -8,6 +8,7 @@
 #define MOZILLA_GFX_COMPOSITOROGL_H
 
 #include <map>
+#include <unordered_map>
 #include <unordered_set>
 
 #include "gfx2DGlue.h"
@@ -36,6 +37,14 @@
 #include "nsXULAppAPI.h"      // for XRE_GetProcessType
 #include "nscore.h"           // for NS_IMETHOD
 
+#ifdef XP_MACOSX
+// This file uses IOSurfacePtr instead of IOSurfaceRef because IOSurfaceRef is
+// hard to forward declare, and including <IOSurface/IOSurface.h> brings in
+// MacTypes.h which defines Point and Rect which cause name lookup trouble.
+struct DummyIOSurface;
+typedef DummyIOSurface* IOSurfacePtr;
+#endif
+
 class nsIWidget;
 
 namespace mozilla {
@@ -269,6 +278,11 @@ class CompositorOGL final : public Compositor {
   void RegisterTextureSource(TextureSource* aTextureSource);
   void UnregisterTextureSource(TextureSource* aTextureSource);
 
+#ifdef XP_MACOSX
+  void RegisterIOSurface(IOSurfacePtr aSurface);
+  void UnregisterIOSurface(IOSurfacePtr aSurface);
+#endif
+
  private:
   template <typename Geometry>
   void DrawGeometry(const Geometry& aGeometry, const gfx::Rect& aRect,
@@ -476,6 +490,18 @@ class CompositorOGL final : public Compositor {
   gfx::IntSize mViewportSize;
 
   ShaderProgramOGL* mCurrentProgram;
+
+#ifdef XP_MACOSX
+  struct IOSurfaceRefHasher {
+    std::size_t operator()(const IOSurfacePtr& aSurface) const {
+      return HashGeneric(reinterpret_cast<uintptr_t>(aSurface));
+    }
+  };
+
+  std::unordered_map<IOSurfacePtr, RefPtr<CompositingRenderTargetOGL>,
+                     IOSurfaceRefHasher>
+      mRegisteredIOSurfaceRenderTargets;
+#endif
 };
 
 }  // namespace layers