Bug 1298058 - Use different lock/unlock function for WebVR composition path. r=milan

gfx/layers/composite/TextureHost.h

@@ -394,12 +394,21 @@ public:
    * Lock the texture host for compositing.
    */
   virtual bool Lock() { return true; }
-
   /**
-   * Unlock the texture host after compositing.
+   * Unlock the texture host after compositing. Lock() and Unlock() should be
+   * called in pair.
    */
   virtual void Unlock() {}
 
+  /**
+   * Lock the texture host for compositing without using compositor.
+   */
+  virtual bool LockWithoutCompositor() { return true; }
+  /**
+   * Similar to Unlock(), but it should be called with LockWithoutCompositor().
+   */
+  virtual void UnlockWithoutCompositor() {}
+
   /**
    * Note that the texture host format can be different from its corresponding
    * texture source's. For example a ShmemTextureHost can have the ycbcr
@@ -811,6 +820,29 @@ private:
   bool mLocked;
 };
 
+class MOZ_STACK_CLASS AutoLockTextureHostWithoutCompositor
+{
+public:
+  explicit AutoLockTextureHostWithoutCompositor(TextureHost* aTexture)
+    : mTexture(aTexture)
+  {
+    mLocked = mTexture ? mTexture->LockWithoutCompositor() : false;
+  }
+
+  ~AutoLockTextureHostWithoutCompositor()
+  {
+    if (mTexture && mLocked) {
+      mTexture->UnlockWithoutCompositor();
+    }
+  }
+
+  bool Failed() { return mTexture && !mLocked; }
+
+private:
+  RefPtr<TextureHost> mTexture;
+  bool mLocked;
+};
+
 /**
  * This can be used as an offscreen rendering target by the compositor, and
  * subsequently can be used as a source by the compositor.

gfx/layers/d3d11/TextureD3D11.cpp

@@ -733,11 +733,40 @@ DXGITextureHostD3D11::GetCompositor()
 bool
 DXGITextureHostD3D11::Lock()
 {
-  /**
-   * Note: This function may be called when mCompositor is null
-   *       such as during WebVR frame submission.
-   **/
+  if (!mCompositor) {
+    // Make an early return here if we call SetCompositor() with an incompatible
+    // compositor. This check tries to prevent the problem where we use that
+    // incompatible compositor to compose this texture.
+    return false;
+  }
 
+  return LockInternal();
+}
+
+bool
+DXGITextureHostD3D11::LockWithoutCompositor()
+{
+  // Unlike the normal Lock() function, this function may be called when
+  // mCompositor is nullptr such as during WebVR frame submission. So, there is
+  // no 'mCompositor' checking here.
+  return LockInternal();
+}
+
+void
+DXGITextureHostD3D11::Unlock()
+{
+  UnlockInternal();
+}
+
+void
+DXGITextureHostD3D11::UnlockWithoutCompositor()
+{
+  UnlockInternal();
+}
+
+bool
+DXGITextureHostD3D11::LockInternal()
+{
   if (!GetDevice()) {
     NS_WARNING("trying to lock a TextureHost without a D3D device");
     return false;
@@ -758,7 +787,7 @@ DXGITextureHostD3D11::Lock()
 }
 
 void
-DXGITextureHostD3D11::Unlock()
+DXGITextureHostD3D11::UnlockInternal()
 {
   UnlockD3DTexture(mTextureSource->GetD3D11Texture());
 }

gfx/layers/d3d11/TextureD3D11.h

@@ -296,9 +296,11 @@ public:
   virtual gfx::SurfaceFormat GetFormat() const override { return mFormat; }
 
   virtual bool Lock() override;
-
   virtual void Unlock() override;
 
+  virtual bool LockWithoutCompositor() override;
+  virtual void UnlockWithoutCompositor() override;
+
   virtual gfx::IntSize GetSize() const override { return mSize; }
 
   virtual already_AddRefed<gfx::DataSourceSurface> GetAsSurface() override
@@ -307,6 +309,9 @@ public:
   }
 
 protected:
+  bool LockInternal();
+  void UnlockInternal();
+
   RefPtr<ID3D11Device> GetDevice();
 
   bool OpenSharedHandle();

gfx/vr/VRDisplayHost.cpp

@@ -83,7 +83,9 @@ VRDisplayHost::SubmitFrame(VRLayerParent* aLayer, const int32_t& aInputFrameID,
   // compensate.
 
   TextureHost* th = TextureHost::AsTextureHost(aTexture);
-  AutoLockTextureHost autoLock(th);
+  // WebVR doesn't use the compositor to compose the frame, so use
+  // AutoLockTextureHostWithoutCompositor here.
+  AutoLockTextureHostWithoutCompositor autoLock(th);
   if (autoLock.Failed()) {
     NS_WARNING("Failed to lock the VR layer texture");
     return;