Bug 1343730 - Part 1: Support submitFrame and encode the frame as a base64 image in VRPuppet; r=kip

MozReview-Commit-ID: jHKHSoNo6X

--HG--
extra : rebase_source : df5050a488e19af81ebd240c5fb5fdfd30b46907

gfx/thebes/gfxPrefs.h

@@ -357,6 +357,7 @@ private:
   DECL_GFX_PREF(Live, "dom.vr.poseprediction.enabled",         VRPosePredictionEnabled, bool, true);
   DECL_GFX_PREF(Live, "dom.vr.require-gesture",                VRRequireGesture, bool, true);
   DECL_GFX_PREF(Live, "dom.vr.puppet.enabled",                 VRPuppetEnabled, bool, false);
+  DECL_GFX_PREF(Live, "dom.vr.puppet.submitframe",             VRPuppetSubmitFrame, uint32_t, 0);
   DECL_GFX_PREF(Live, "dom.w3c_pointer_events.enabled",        PointerEventsEnabled, bool, false);
   DECL_GFX_PREF(Live, "dom.w3c_touch_events.enabled",          TouchEventsEnabled, int32_t, 0);
 

gfx/vr/VRDisplayClient.cpp

@@ -148,3 +148,15 @@ VRDisplayClient::NotifyDisconnected()
 {
   mDisplayInfo.mIsConnected = false;
 }
+
+void
+VRDisplayClient::UpdateSubmitFrameResult(const VRSubmitFrameResultInfo& aResult)
+{
+  mSubmitFrameResult = aResult;
+}
+
+void
+VRDisplayClient::GetSubmitFrameResult(VRSubmitFrameResultInfo& aResult)
+{
+  aResult = mSubmitFrameResult;
+}

gfx/vr/VRDisplayClient.h

@@ -26,9 +26,11 @@ public:
   explicit VRDisplayClient(const VRDisplayInfo& aDisplayInfo);
 
   void UpdateDisplayInfo(const VRDisplayInfo& aDisplayInfo);
+  void UpdateSubmitFrameResult(const VRSubmitFrameResultInfo& aResult);
 
   const VRDisplayInfo& GetDisplayInfo() const { return mDisplayInfo; }
   virtual VRHMDSensorState GetSensorState();
+  void GetSubmitFrameResult(VRSubmitFrameResultInfo& aResult);
 
   virtual void ZeroSensor();
 
@@ -53,6 +55,9 @@ protected:
 
   TimeStamp mLastVSyncTime;
   int mPresentationCount;
+
+private:
+  VRSubmitFrameResultInfo mSubmitFrameResult;
 };
 
 } // namespace gfx

gfx/vr/VRManager.cpp

@@ -468,5 +468,13 @@ VRManager::NotifyVibrateHapticCompleted(uint32_t aPromiseID)
   }
 }
 
+void
+VRManager::DispatchSubmitFrameResult(uint32_t aDisplayID, const VRSubmitFrameResultInfo& aResult)
+{
+  for (auto iter = mVRManagerParents.Iter(); !iter.Done(); iter.Next()) {
+    Unused << iter.Get()->GetKey()->SendDispatchSubmitFrameResult(aDisplayID, aResult);
+  }
+}
+
 } // namespace gfx
 } // namespace mozilla

gfx/vr/VRManager.h

@@ -53,6 +53,7 @@ public:
                      double aIntensity, double aDuration, uint32_t aPromiseID);
   void StopVibrateHaptic(uint32_t aControllerIdx);
   void NotifyVibrateHapticCompleted(uint32_t aPromiseID);
+  void DispatchSubmitFrameResult(uint32_t aDisplayID, const VRSubmitFrameResultInfo& aResult);
 
 protected:
   VRManager();

gfx/vr/gfxVR.h

@@ -209,6 +209,21 @@ struct VRHMDSensorState {
   }
 };
 
+struct VRSubmitFrameResultInfo
+{
+  VRSubmitFrameResultInfo()
+   : mFrameNum(0),
+     mWidth(0),
+     mHeight(0)
+  {}
+
+  nsCString mBase64Image;
+  SurfaceFormat mFormat;
+  uint32_t mFrameNum;
+  uint32_t mWidth;
+  uint32_t mHeight;
+};
+
 struct VRControllerInfo
 {
   VRDeviceType GetType() const { return mType; }

gfx/vr/gfxVRPuppet.cpp

@@ -6,19 +6,31 @@
 #if defined(XP_WIN)
 #include "CompositorD3D11.h"
 #include "TextureD3D11.h"
+#include "mozilla/gfx/DeviceManagerDx.h"
 #endif // XP_WIN
 
+#include "mozilla/Base64.h"
+#include "mozilla/gfx/DataSurfaceHelpers.h"
+#include "gfxUtils.h"
 #include "gfxVRPuppet.h"
 
 #include "mozilla/dom/GamepadEventTypes.h"
 #include "mozilla/dom/GamepadBinding.h"
 
+// See CompositorD3D11Shaders.h
+namespace mozilla {
+namespace layers {
+struct ShaderBytes { const void* mData; size_t mLength; };
+extern ShaderBytes sRGBShader;
+extern ShaderBytes sLayerQuadVS;
+} // namespace layers
+} // namespace mozilla
+
 using namespace mozilla;
 using namespace mozilla::gfx;
 using namespace mozilla::gfx::impl;
 using namespace mozilla::layers;
 
-
 // Reminder: changing the order of these buttons may break web content
 static const uint64_t kPuppetButtonMask[] = {
   1,
@@ -44,6 +56,7 @@ static const uint32_t kNumPuppetHaptcs = 1;
 VRDisplayPuppet::VRDisplayPuppet()
  : VRDisplayHost(VRDeviceType::Puppet)
  , mIsPresenting(false)
+ , mFrameNum(0)
 {
   MOZ_COUNT_CTOR_INHERITED(VRDisplayPuppet, VRDisplayHost);
 
@@ -163,6 +176,83 @@ VRDisplayPuppet::StartPresentation()
     return;
   }
   mIsPresenting = true;
+
+#if defined(XP_WIN)
+  if (!mDevice) {
+    mDevice = gfx::DeviceManagerDx::Get()->GetCompositorDevice();
+    if (!mDevice) {
+      NS_WARNING("Failed to get a D3D11Device for Puppet");
+      return;
+    }
+  }
+
+  mDevice->GetImmediateContext(getter_AddRefs(mContext));
+  if (!mContext) {
+    NS_WARNING("Failed to get immediate context for Puppet");
+    return;
+  }
+
+  if (FAILED(mDevice->CreateVertexShader(sLayerQuadVS.mData,
+                      sLayerQuadVS.mLength, nullptr, &mQuadVS))) {
+    NS_WARNING("Failed to create vertex shader for Puppet");
+    return;
+  }
+
+  if (FAILED(mDevice->CreatePixelShader(sRGBShader.mData,
+                      sRGBShader.mLength, nullptr, &mQuadPS))) {
+    NS_WARNING("Failed to create pixel shader for Puppet");
+    return;
+  }
+
+  CD3D11_BUFFER_DESC cBufferDesc(sizeof(layers::VertexShaderConstants),
+                                 D3D11_BIND_CONSTANT_BUFFER,
+                                 D3D11_USAGE_DYNAMIC,
+                                 D3D11_CPU_ACCESS_WRITE);
+
+  if (FAILED(mDevice->CreateBuffer(&cBufferDesc, nullptr, getter_AddRefs(mVSConstantBuffer)))) {
+    NS_WARNING("Failed to vertex shader constant buffer for Puppet");
+    return;
+  }
+
+  cBufferDesc.ByteWidth = sizeof(layers::PixelShaderConstants);
+  if (FAILED(mDevice->CreateBuffer(&cBufferDesc, nullptr, getter_AddRefs(mPSConstantBuffer)))) {
+    NS_WARNING("Failed to pixel shader constant buffer for Puppet");
+    return;
+  }
+
+  CD3D11_SAMPLER_DESC samplerDesc(D3D11_DEFAULT);
+  if (FAILED(mDevice->CreateSamplerState(&samplerDesc, getter_AddRefs(mLinearSamplerState)))) {
+    NS_WARNING("Failed to create sampler state for Puppet");
+    return;
+  }
+
+  D3D11_INPUT_ELEMENT_DESC layout[] =
+  {
+    { "POSITION", 0, DXGI_FORMAT_R32G32_FLOAT, 0, 0, D3D11_INPUT_PER_VERTEX_DATA, 0 },
+  };
+
+  if (FAILED(mDevice->CreateInputLayout(layout,
+                                        sizeof(layout) / sizeof(D3D11_INPUT_ELEMENT_DESC),
+                                        sLayerQuadVS.mData,
+                                        sLayerQuadVS.mLength,
+                                        getter_AddRefs(mInputLayout)))) {
+    NS_WARNING("Failed to create input layout for Puppet");
+    return;
+  }
+
+  Vertex vertices[] = { { { 0.0, 0.0 } },{ { 1.0, 0.0 } },{ { 0.0, 1.0 } },{ { 1.0, 1.0 } } };
+  CD3D11_BUFFER_DESC bufferDesc(sizeof(vertices), D3D11_BIND_VERTEX_BUFFER);
+  D3D11_SUBRESOURCE_DATA data;
+  data.pSysMem = (void*)vertices;
+
+  if (FAILED(mDevice->CreateBuffer(&bufferDesc, &data, getter_AddRefs(mVertexBuffer)))) {
+    NS_WARNING("Failed to create vertex buffer for Puppet");
+    return;
+  }
+
+  memset(&mVSConstants, 0, sizeof(mVSConstants));
+  memset(&mPSConstants, 0, sizeof(mPSConstants));
+#endif // XP_WIN
 }
 
 void
@@ -176,6 +266,35 @@ VRDisplayPuppet::StopPresentation()
 }
 
 #if defined(XP_WIN)
+bool
+VRDisplayPuppet::UpdateConstantBuffers()
+{
+  HRESULT hr;
+  D3D11_MAPPED_SUBRESOURCE resource;
+  resource.pData = nullptr;
+
+  hr = mContext->Map(mVSConstantBuffer, 0, D3D11_MAP_WRITE_DISCARD, 0, &resource);
+  if (FAILED(hr) || !resource.pData) {
+    return false;
+  }
+  *(VertexShaderConstants*)resource.pData = mVSConstants;
+  mContext->Unmap(mVSConstantBuffer, 0);
+  resource.pData = nullptr;
+
+  hr = mContext->Map(mPSConstantBuffer, 0, D3D11_MAP_WRITE_DISCARD, 0, &resource);
+  if (FAILED(hr) || !resource.pData) {
+    return false;
+  }
+  *(PixelShaderConstants*)resource.pData = mPSConstants;
+  mContext->Unmap(mPSConstantBuffer, 0);
+
+  ID3D11Buffer *buffer = mVSConstantBuffer;
+  mContext->VSSetConstantBuffers(0, 1, &buffer);
+  buffer = mPSConstantBuffer;
+  mContext->PSSetConstantBuffers(0, 1, &buffer);
+  return true;
+}
+
 void
 VRDisplayPuppet::SubmitFrame(TextureSourceD3D11* aSource,
                              const IntSize& aSize,
@@ -187,16 +306,160 @@ VRDisplayPuppet::SubmitFrame(TextureSourceD3D11* aSource,
     return;
   }
 
-  ID3D11Texture2D* tex = aSource->GetD3D11Texture();
-  MOZ_ASSERT(tex);
-
-  // TODO: Bug 1343730, Need to block until the next simulated
-  // vblank interval and capture frames for use in reftests.
-
-  // Trigger the next VSync immediately
   VRManager *vm = VRManager::Get();
   MOZ_ASSERT(vm);
+
+  switch (gfxPrefs::VRPuppetSubmitFrame()) {
+    case 0:
+      // The VR frame is not displayed.
+      break;
+    case 1:
+    {
+      // The frames are submitted to VR compositor are decoded
+      // into a base64Image and dispatched to the DOM side.
+      D3D11_TEXTURE2D_DESC desc;
+      ID3D11Texture2D* texture = aSource->GetD3D11Texture();
+      texture->GetDesc(&desc);
+      DXGI_FORMAT format = desc.Format;
+      // Map the staging resource
+      ID3D11Texture2D* mappedTexture = nullptr;
+      D3D11_MAPPED_SUBRESOURCE mapInfo;
+      HRESULT hr = mContext->Map(texture,
+                                 0,  // Subsource
+                                 D3D11_MAP_READ,
+                                 0,  // MapFlags
+                                 &mapInfo);
+
+      if (FAILED(hr)) {
+        // If we can't map this texture, copy it to a staging resource.
+        if (hr == E_INVALIDARG) {
+          D3D11_TEXTURE2D_DESC desc2;
+          desc2.Width = desc.Width;
+          desc2.Height = desc.Height;
+          desc2.MipLevels = desc.MipLevels;
+          desc2.ArraySize = desc.ArraySize;
+          desc2.Format = desc.Format;
+          desc2.SampleDesc = desc.SampleDesc;
+          desc2.Usage = D3D11_USAGE_STAGING;
+          desc2.BindFlags = 0;
+          desc2.CPUAccessFlags = D3D11_CPU_ACCESS_READ;
+          desc2.MiscFlags = 0;
+
+          ID3D11Texture2D* stagingTexture = nullptr;
+          hr = mDevice->CreateTexture2D(&desc2, nullptr, &stagingTexture);
+          if (FAILED(hr)) {
+            MOZ_ASSERT(false, "Failed to create a staging texture");
+            return;
+          }
+          // Copy the texture to a staging resource
+          mContext->CopyResource(stagingTexture, texture);
+          // Map the staging resource
+          hr = mContext->Map(stagingTexture,
+                             0,  // Subsource
+                             D3D11_MAP_READ,
+                             0,  // MapFlags
+                             &mapInfo);
+          if (FAILED(hr)) {
+            MOZ_ASSERT(false, "Failed to map staging texture");
+          }
+          mappedTexture = stagingTexture;
+        }
+        else {
+          MOZ_ASSERT(false, "Failed to map staging texture");
+          return;
+        }
+      } else {
+        mappedTexture = texture;
+      }
+      // Ideally, we should convert the srcData to a PNG image and decode it
+      // to a Base64 string here, but the GPU process does not have the privilege to
+      // access the image library. So, we have to convert the RAW image data
+      // to a base64 string and forward it to let the content process to
+      // do the image conversion.
+      char* srcData = static_cast<char*>(mapInfo.pData);
+      VRSubmitFrameResultInfo result;
+      result.mFormat = SurfaceFormat::B8G8R8A8;
+      result.mWidth = desc.Width;
+      result.mHeight = desc.Height;
+      result.mFrameNum = mFrameNum;
+      nsCString rawString(Substring((char*)srcData, mapInfo.RowPitch * desc.Height));
+
+      if (Base64Encode(rawString, result.mBase64Image) != NS_OK) {
+        MOZ_ASSERT(false, "Failed to encode base64 images.");
+      }
+      mContext->Unmap(mappedTexture, 0);
+      // Dispatch the base64 encoded string to the DOM side, and it will be decoded
+      // and convert to a PNG image there.
+      vm->DispatchSubmitFrameResult(mDisplayInfo.mDisplayID, result);
+      break;
+    }
+    case 2:
+    {
+      // The VR compositor sumbmit frame to the screen window,
+      // the current coordinate is at (0, 0, width, height).
+      Matrix viewMatrix = Matrix::Translation(-1.0, 1.0);
+      viewMatrix.PreScale(2.0f / float(aSize.width), 2.0f / float(aSize.height));
+      viewMatrix.PreScale(1.0f, -1.0f);
+      Matrix4x4 projection = Matrix4x4::From2D(viewMatrix);
+      projection._33 = 0.0f;
+
+      Matrix transform2d;
+      gfx::Matrix4x4 transform = gfx::Matrix4x4::From2D(transform2d);
+
+      const float posX = 0.0f, posY = 0.0f;
+      D3D11_VIEWPORT viewport;
+      viewport.MinDepth = 0.0f;
+      viewport.MaxDepth = 1.0f;
+      viewport.Width = aSize.width;
+      viewport.Height = aSize.height;
+      viewport.TopLeftX = posX;
+      viewport.TopLeftY = posY;
+
+      D3D11_RECT scissor;
+      scissor.left = posX;
+      scissor.right = aSize.width + posX;
+      scissor.top = posY;
+      scissor.bottom = aSize.height + posY;
+
+      memcpy(&mVSConstants.layerTransform, &transform._11, sizeof(mVSConstants.layerTransform));
+      memcpy(&mVSConstants.projection, &projection._11, sizeof(mVSConstants.projection));
+      mVSConstants.renderTargetOffset[0] = 0.0f;
+      mVSConstants.renderTargetOffset[1] = 0.0f;
+      mVSConstants.layerQuad = Rect(0.0f, 0.0f, aSize.width, aSize.height);
+      mVSConstants.textureCoords = Rect(0.0f, 1.0f, 1.0f, -1.0f);
+
+      mPSConstants.layerOpacity[0] = 1.0f;
+
+      ID3D11Buffer* vbuffer = mVertexBuffer;
+      UINT vsize = sizeof(Vertex);
+      UINT voffset = 0;
+      mContext->IASetVertexBuffers(0, 1, &vbuffer, &vsize, &voffset);
+      mContext->IASetIndexBuffer(nullptr, DXGI_FORMAT_R16_UINT, 0);
+      mContext->IASetInputLayout(mInputLayout);
+      mContext->RSSetViewports(1, &viewport);
+      mContext->RSSetScissorRects(1, &scissor);
+      mContext->IASetPrimitiveTopology(D3D11_PRIMITIVE_TOPOLOGY_TRIANGLESTRIP);
+      mContext->VSSetShader(mQuadVS, nullptr, 0);
+      mContext->PSSetShader(mQuadPS, nullptr, 0);
+      ID3D11ShaderResourceView* srView = aSource->GetShaderResourceView();
+      mContext->PSSetShaderResources(0 /* 0 == TexSlot::RGB */, 1, &srView);
+      // XXX Use Constant from TexSlot in CompositorD3D11.cpp?
+
+      ID3D11SamplerState *sampler = mLinearSamplerState;
+      mContext->PSSetSamplers(0, 1, &sampler);
+
+      if (!UpdateConstantBuffers()) {
+        NS_WARNING("Failed to update constant buffers for Puppet");
+        return;
+      }
+      mContext->Draw(4, 0);
+      break;
+    }
+  }
+
+  // Trigger the next VSync immediately
   vm->NotifyVRVsync(mDisplayInfo.mDisplayID);
+  ++mFrameNum;
 }
 #else
 void
@@ -217,6 +480,7 @@ VRDisplayPuppet::SubmitFrame(TextureSourceOGL* aSource,
   VRManager *vm = VRManager::Get();
   MOZ_ASSERT(vm);
   vm->NotifyVRVsync(mDisplayInfo.mDisplayID);
+  ++mFrameNum;
 }
 #endif
 

gfx/vr/gfxVRPuppet.h

@@ -53,7 +53,24 @@ protected:
   bool mIsPresenting;
 
 private:
+#if defined(XP_WIN)
+  bool UpdateConstantBuffers();
+
+  RefPtr<ID3D11Device> mDevice;
+  RefPtr<ID3D11DeviceContext> mContext;
+  ID3D11VertexShader* mQuadVS;
+  ID3D11PixelShader* mQuadPS;
+  RefPtr<ID3D11SamplerState> mLinearSamplerState;
+  layers::VertexShaderConstants mVSConstants;
+  layers::PixelShaderConstants mPSConstants;
+  RefPtr<ID3D11Buffer> mVSConstantBuffer;
+  RefPtr<ID3D11Buffer> mPSConstantBuffer;
+  RefPtr<ID3D11Buffer> mVertexBuffer;
+  RefPtr<ID3D11InputLayout> mInputLayout;
+#endif
+
   VRHMDSensorState mSensorState;
+  uint32_t mFrameNum;
 };
 
 class VRControllerPuppet : public VRControllerHost

gfx/vr/ipc/PVRManager.ipdl

@@ -18,6 +18,7 @@ using struct mozilla::gfx::VRDisplayInfo from "gfxVR.h";
 using struct mozilla::gfx::VRSensorUpdate from "gfxVR.h";
 using struct mozilla::gfx::VRHMDSensorState from "gfxVR.h";
 using struct mozilla::gfx::VRControllerInfo from "gfxVR.h";
+using struct mozilla::gfx::VRSubmitFrameResultInfo from "gfxVR.h";
 using mozilla::layers::LayersBackend from "mozilla/layers/LayersTypes.h";
 using mozilla::layers::TextureFlags from "mozilla/layers/CompositorTypes.h";
 
@@ -82,6 +83,7 @@ child:
 
   async NotifyVSync();
   async NotifyVRVSync(uint32_t aDisplayID);
+  async DispatchSubmitFrameResult(uint32_t aDisplayID, VRSubmitFrameResultInfo aResult);
   async GamepadUpdate(GamepadChangeEvent aGamepadEvent);
   async ReplyGamepadVibrateHaptic(uint32_t aPromiseID);
 

gfx/vr/ipc/VRManagerChild.cpp

@@ -718,5 +718,18 @@ VRManagerChild::RecvReplyGamepadVibrateHaptic(const uint32_t& aPromiseID)
   return IPC_OK();
 }
 
+mozilla::ipc::IPCResult
+VRManagerChild::RecvDispatchSubmitFrameResult(const uint32_t& aDisplayID,
+                                              const VRSubmitFrameResultInfo& aResult)
+{
+   for (auto& display : mDisplays) {
+    if (display->GetDisplayInfo().GetDisplayID() == aDisplayID) {
+      display->UpdateSubmitFrameResult(aResult);
+    }
+  }
+
+  return IPC_OK();
+}
+
 } // namespace gfx
 } // namespace mozilla

gfx/vr/ipc/VRManagerChild.h

@@ -128,6 +128,7 @@ protected:
 
   virtual mozilla::ipc::IPCResult RecvNotifyVSync() override;
   virtual mozilla::ipc::IPCResult RecvNotifyVRVSync(const uint32_t& aDisplayID) override;
+  virtual mozilla::ipc::IPCResult RecvDispatchSubmitFrameResult(const uint32_t& aDisplayID, const VRSubmitFrameResultInfo& aResult) override;
   virtual mozilla::ipc::IPCResult RecvGamepadUpdate(const GamepadChangeEvent& aGamepadEvent) override;
   virtual mozilla::ipc::IPCResult RecvReplyGamepadVibrateHaptic(const uint32_t& aPromiseID) override;
 

gfx/vr/ipc/VRMessageUtils.h

@@ -190,6 +190,35 @@ struct ParamTraits<mozilla::gfx::VRControllerInfo>
     return true;
   }
 };
+
+template <>
+struct ParamTraits<mozilla::gfx::VRSubmitFrameResultInfo>
+{
+  typedef mozilla::gfx::VRSubmitFrameResultInfo paramType;
+
+  static void Write(Message* aMsg, const paramType& aParam)
+  {
+    WriteParam(aMsg, aParam.mBase64Image);
+    WriteParam(aMsg, aParam.mFormat);
+    WriteParam(aMsg, aParam.mWidth);
+    WriteParam(aMsg, aParam.mHeight);
+    WriteParam(aMsg, aParam.mFrameNum);
+  }
+
+  static bool Read(const Message* aMsg, PickleIterator* aIter, paramType* aResult)
+  {
+    if (!ReadParam(aMsg, aIter, &(aResult->mBase64Image)) ||
+        !ReadParam(aMsg, aIter, &(aResult->mFormat)) ||
+        !ReadParam(aMsg, aIter, &(aResult->mWidth)) ||
+        !ReadParam(aMsg, aIter, &(aResult->mHeight)) ||
+        !ReadParam(aMsg, aIter, &(aResult->mFrameNum))) {
+      return false;
+    }
+
+    return true;
+  }
+};
+
 } // namespace IPC
 
 #endif // mozilla_gfx_vr_VRMessageUtils_h

modules/libpref/init/all.js

@@ -5126,6 +5126,10 @@ pref("gfx.vr.osvr.clientLibPath", "");
 pref("gfx.vr.osvr.clientKitLibPath", "");
 // Puppet device, used for simulating VR hardware within tests and dev tools
 pref("dom.vr.puppet.enabled", false);
+// Allow displaying the result of vr submitframe (0: disable, 1: store the
+// result as a base64 image, 2: show it on the screen).
+pref("dom.vr.puppet.submitframe", 0);
+// VR test system.
 pref("dom.vr.test.enabled", false);
 // MMS UA Profile settings
 pref("wap.UAProf.url", "");