Bug 1495025 - P1. Search for alternative pixel format when stream change. r=cpearce

When decoding a vp9 profile 2 (10 bits), the MF_E_TRANSFORM_STREAM_CHANGE message is returned. We need to look for alternative format type other than NV12: 10/16 bits.

When using those formats, we can no longer assume that the D3D11ShareHandleImage can use NV12. So we will convert to RGBA32 on the fly via a MFT.

Differential Revision: https://phabricator.services.mozilla.com/D7294

dom/media/platforms/wmf/DXVA2Manager.cpp

@@ -623,7 +623,7 @@ public:
   virtual HRESULT CopyToBGRATexture(ID3D11Texture2D *aInTexture,
                                     ID3D11Texture2D** aOutTexture);
 
-  HRESULT ConfigureForSize(uint32_t aWidth, uint32_t aHeight) override;
+  HRESULT ConfigureForSize(IMFMediaType* aInputType) override;
 
   bool IsD3D11() override { return true; }
 
@@ -652,7 +652,7 @@ private:
   uint32_t mWidth = 0;
   uint32_t mHeight = 0;
   UINT mDeviceManagerToken = 0;
-  bool mConfiguredForSize = false;
+  RefPtr<IMFMediaType> mInputType;
 };
 
 bool
@@ -921,14 +921,32 @@ D3D11DXVA2Manager::CopyToImage(IMFSample* aVideoSample,
 
   RefPtr<D3D11ShareHandleImage> image =
     new D3D11ShareHandleImage(gfx::IntSize(mWidth, mHeight), aRegion);
-  bool ok = image->AllocateTexture(mTextureClientAllocator, mDevice);
+
+  // Retrieve the DXGI_FORMAT for the current video sample.
+  RefPtr<IMFMediaBuffer> buffer;
+  HRESULT hr = aVideoSample->GetBufferByIndex(0, getter_AddRefs(buffer));
+  NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
+
+  RefPtr<IMFDXGIBuffer> dxgiBuf;
+  hr = buffer->QueryInterface((IMFDXGIBuffer**)getter_AddRefs(dxgiBuf));
+  NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
+
+  RefPtr<ID3D11Texture2D> tex;
+  hr = dxgiBuf->GetResource(__uuidof(ID3D11Texture2D), getter_AddRefs(tex));
+  NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
+
+  D3D11_TEXTURE2D_DESC desc;
+  tex->GetDesc(&desc);
+
+  bool ok = image->AllocateTexture(
+    mTextureClientAllocator, mDevice, desc.Format == DXGI_FORMAT_NV12);
   NS_ENSURE_TRUE(ok, E_FAIL);
 
-  RefPtr<TextureClient> client = image->GetTextureClient(ImageBridgeChild::GetSingleton().get());
+  RefPtr<TextureClient> client =
+    image->GetTextureClient(ImageBridgeChild::GetSingleton().get());
   NS_ENSURE_TRUE(client, E_FAIL);
 
   RefPtr<IDXGIKeyedMutex> mutex;
-  HRESULT hr = S_OK;
   RefPtr<ID3D11Texture2D> texture = image->GetTexture();
 
   texture->QueryInterface((IDXGIKeyedMutex**)getter_AddRefs(mutex));
@@ -948,23 +966,10 @@ D3D11DXVA2Manager::CopyToImage(IMFSample* aVideoSample,
       // to create a copy of that frame as a sharable resource, save its share
       // handle, and put that handle into the rendering pipeline.
 
-      RefPtr<IMFMediaBuffer> buffer;
-      hr = aVideoSample->GetBufferByIndex(0, getter_AddRefs(buffer));
-      NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
-
-      RefPtr<IMFDXGIBuffer> dxgiBuf;
-      hr = buffer->QueryInterface((IMFDXGIBuffer**)getter_AddRefs(dxgiBuf));
-      NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
-
-      RefPtr<ID3D11Texture2D> tex;
-      hr = dxgiBuf->GetResource(__uuidof(ID3D11Texture2D), getter_AddRefs(tex));
-      NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
-
       UINT index;
       dxgiBuf->GetSubresourceIndex(&index);
       mContext->CopySubresourceRegion(texture, 0, 0, 0, 0, tex, index, nullptr);
     } else {
-      // Our video sample is in NV12 format but our output texture is in BGRA.
       // Use MFT to do color conversion.
       hr = E_FAIL;
       mozilla::mscom::EnsureMTA(
@@ -996,6 +1001,21 @@ D3D11DXVA2Manager::CopyToImage(IMFSample* aVideoSample,
   return S_OK;
 }
 
+const GUID& TextureFormatToSubType(DXGI_FORMAT aFormat)
+{
+  switch (aFormat) {
+    case DXGI_FORMAT_NV12:
+      return MFVideoFormat_NV12;
+    case DXGI_FORMAT_P010:
+      return MFVideoFormat_P010;
+    case DXGI_FORMAT_P016:
+      return MFVideoFormat_P016;
+    default:
+      // TextureFormat not handled.
+      return MFVideoFormat_NV12;
+  }
+}
+
 HRESULT
 D3D11DXVA2Manager::CopyToBGRATexture(ID3D11Texture2D *aInTexture,
                                      ID3D11Texture2D** aOutTexture)
@@ -1010,8 +1030,32 @@ D3D11DXVA2Manager::CopyToBGRATexture(ID3D11Texture2D *aInTexture,
 
   CD3D11_TEXTURE2D_DESC desc;
   aInTexture->GetDesc(&desc);
-  hr = ConfigureForSize(desc.Width, desc.Height);
-  NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
+
+  if (!mInputType || desc.Width != mWidth || desc.Height != mHeight) {
+    RefPtr<IMFMediaType> inputType;
+    hr = wmf::MFCreateMediaType(getter_AddRefs(inputType));
+    NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
+
+    hr = inputType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
+    NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
+
+    hr = inputType->SetGUID(MF_MT_SUBTYPE, TextureFormatToSubType(desc.Format));
+    NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
+
+    hr =
+      MFSetAttributeSize(inputType, MF_MT_FRAME_SIZE, desc.Width, desc.Height);
+    NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
+
+    hr =
+      inputType->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive);
+    NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
+
+    hr = inputType->SetUINT32(MF_MT_ALL_SAMPLES_INDEPENDENT, TRUE);
+    NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
+
+    hr = ConfigureForSize(inputType);
+    NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
+  }
 
   RefPtr<IDXGIKeyedMutex> mutex;
   inTexture->QueryInterface((IDXGIKeyedMutex**)getter_AddRefs(mutex));
@@ -1090,31 +1134,19 @@ HRESULT ConfigureOutput(IMFMediaType* aOutput, void* aData)
 }
 
 HRESULT
-D3D11DXVA2Manager::ConfigureForSize(uint32_t aWidth, uint32_t aHeight)
+D3D11DXVA2Manager::ConfigureForSize(IMFMediaType* aInputType)
 {
-  if (mConfiguredForSize && aWidth == mWidth && aHeight == mHeight) {
-    // If the size hasn't changed, don't reconfigure.
+
+  if (aInputType == mInputType) {
+    // If the media type hasn't changed, don't reconfigure.
     return S_OK;
   }
 
-  mWidth = aWidth;
-  mHeight = aHeight;
-
-  RefPtr<IMFMediaType> inputType;
-  HRESULT hr = wmf::MFCreateMediaType(getter_AddRefs(inputType));
-  NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
-
-  hr = inputType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
-  NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
-
-  hr = inputType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_NV12);
-  NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
-
-  hr = inputType->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive);
-  NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
-
-  hr = inputType->SetUINT32(MF_MT_ALL_SAMPLES_INDEPENDENT, TRUE);
+  UINT32 width = 0, height = 0;
+  HRESULT hr = MFGetAttributeSize(aInputType, MF_MT_FRAME_SIZE, &width, &height);
   NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
+  mWidth = width;
+  mHeight = height;
 
   RefPtr<IMFAttributes> attr;
   mozilla::mscom::EnsureMTA(
@@ -1127,9 +1159,6 @@ D3D11DXVA2Manager::ConfigureForSize(uint32_t aWidth, uint32_t aHeight)
   hr = attr->SetUINT32(MF_LOW_LATENCY, FALSE);
   NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
 
-  hr = MFSetAttributeSize(inputType, MF_MT_FRAME_SIZE, aWidth, aHeight);
-  NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
-
   RefPtr<IMFMediaType> outputType;
   hr = wmf::MFCreateMediaType(getter_AddRefs(outputType));
   NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
@@ -1144,11 +1173,11 @@ D3D11DXVA2Manager::ConfigureForSize(uint32_t aWidth, uint32_t aHeight)
   hr = E_FAIL;
   mozilla::mscom::EnsureMTA([&]() -> void {
     hr =
-      mTransform->SetMediaTypes(inputType, outputType, ConfigureOutput, &size);
+      mTransform->SetMediaTypes(aInputType, outputType, ConfigureOutput, &size);
   });
   NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
 
-  mConfiguredForSize = true;
+  mInputType = aInputType;
 
   return S_OK;
 }

dom/media/platforms/wmf/DXVA2Manager.h

@@ -54,7 +54,7 @@ public:
     return E_FAIL;
   }
 
-  virtual HRESULT ConfigureForSize(uint32_t aWidth, uint32_t aHeight)
+  virtual HRESULT ConfigureForSize(IMFMediaType* aInputType)
   {
     return S_OK;
   }

dom/media/platforms/wmf/MFTDecoder.cpp

@@ -95,7 +95,15 @@ MFTDecoder::SetMediaTypes(IMFMediaType* aInputType,
   HRESULT hr = mDecoder->SetInputType(0, aInputType, 0);
   NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
 
-  hr = SetDecoderOutputType(true /* match all attributes */, aCallback, aData);
+  GUID currentSubtype = {0};
+  hr = aOutputType->GetGUID(MF_MT_SUBTYPE, &currentSubtype);
+  NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
+
+  hr = SetDecoderOutputType(currentSubtype,
+                            aOutputType,
+                            true /* match all attributes */,
+                            aCallback,
+                            aData);
   NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
 
   hr = mDecoder->GetInputStreamInfo(0, &mInputStreamInfo);
@@ -121,16 +129,38 @@ MFTDecoder::GetAttributes()
 }
 
 HRESULT
-MFTDecoder::SetDecoderOutputType(bool aMatchAllAttributes,
+MFTDecoder::FindDecoderOutputType(bool aMatchAllAttributes)
+{
+  MOZ_ASSERT(mscom::IsCurrentThreadMTA());
+  MOZ_ASSERT(mOutputType, "SetDecoderTypes must have been called once");
+
+  GUID currentSubtype = {0};
+  HRESULT hr = mOutputType->GetGUID(MF_MT_SUBTYPE, &currentSubtype);
+  NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
+  return FindDecoderOutputTypeWithSubtype(currentSubtype, aMatchAllAttributes);
+}
+
+HRESULT
+MFTDecoder::FindDecoderOutputTypeWithSubtype(const GUID& aSubType,
+                                             bool aMatchAllAttributes)
+{
+  return SetDecoderOutputType(
+    aSubType, nullptr, aMatchAllAttributes, nullptr, nullptr);
+}
+
+HRESULT
+MFTDecoder::SetDecoderOutputType(const GUID& aSubType,
+                                 IMFMediaType* aTypeToUse,
+                                 bool aMatchAllAttributes,
                                  ConfigureOutputCallback aCallback,
                                  void* aData)
 {
   MOZ_ASSERT(mscom::IsCurrentThreadMTA());
   NS_ENSURE_TRUE(mDecoder != nullptr, E_POINTER);
 
-  GUID currentSubtype = {0};
-  HRESULT hr = mOutputType->GetGUID(MF_MT_SUBTYPE, &currentSubtype);
-  NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
+  if (!aTypeToUse) {
+    aTypeToUse = mOutputType;
+  }
 
   // Iterate the enumerate the output types, until we find one compatible
   // with what we need.
@@ -139,14 +169,14 @@ MFTDecoder::SetDecoderOutputType(bool aMatchAllAttributes,
   while (SUCCEEDED(mDecoder->GetOutputAvailableType(
     0, typeIndex++, getter_AddRefs(outputType)))) {
     GUID outSubtype = {0};
-    hr = outputType->GetGUID(MF_MT_SUBTYPE, &outSubtype);
+    HRESULT hr = outputType->GetGUID(MF_MT_SUBTYPE, &outSubtype);
     NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
 
-    BOOL resultMatch = currentSubtype == outSubtype;
+    BOOL resultMatch = aSubType == outSubtype;
 
     if (resultMatch && aMatchAllAttributes) {
-      hr = mOutputType->Compare(outputType, MF_ATTRIBUTES_MATCH_OUR_ITEMS,
-                                &resultMatch);
+      hr = aTypeToUse->Compare(outputType, MF_ATTRIBUTES_MATCH_OUR_ITEMS,
+                               &resultMatch);
       NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
     }
     if (resultMatch == TRUE) {
@@ -163,6 +193,8 @@ MFTDecoder::SetDecoderOutputType(bool aMatchAllAttributes,
       mMFTProvidesOutputSamples = IsFlagSet(mOutputStreamInfo.dwFlags,
                                             MFT_OUTPUT_STREAM_PROVIDES_SAMPLES);
 
+      mOutputType = outputType;
+
       return S_OK;
     }
     outputType = nullptr;

dom/media/platforms/wmf/MFTDecoder.h

@@ -88,10 +88,17 @@ public:
   // Sends a message to the MFT.
   HRESULT SendMFTMessage(MFT_MESSAGE_TYPE aMsg, ULONG_PTR aData);
 
-  HRESULT SetDecoderOutputType(bool aMatchAllAttributes,
+  HRESULT FindDecoderOutputTypeWithSubtype(const GUID& aSubType,
+                                           bool aMatchAllAttributes);
+  HRESULT FindDecoderOutputType(bool aMatchAllAttributes);
+private:
+  // Will search a suitable MediaType using aTypeToUse if set, if not will
+  // use the current mOutputType.
+  HRESULT SetDecoderOutputType(const GUID& aSubType,
+                               IMFMediaType* aTypeToUse,
+                               bool aMatchAllAttributes,
                                ConfigureOutputCallback aCallback,
                                void* aData);
-private:
   HRESULT CreateOutputSample(RefPtr<IMFSample>* aOutSample);
 
   MFT_INPUT_STREAM_INFO mInputStreamInfo;

dom/media/platforms/wmf/WMFAudioMFTManager.cpp

@@ -237,9 +237,8 @@ WMFAudioMFTManager::Output(int64_t aStreamOffset,
       return hr;
     }
     if (hr == MF_E_TRANSFORM_STREAM_CHANGE) {
-      hr = mDecoder->SetDecoderOutputType(true /* check all attribute */,
-                                          nullptr,
-                                          nullptr);
+      hr = mDecoder->FindDecoderOutputType(false /* check all attribute */);
+      NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
       hr = UpdateOutputType();
       NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
       // Catch infinite loops, but some decoders perform at least 2 stream

dom/media/platforms/wmf/WMFVideoMFTManager.cpp

@@ -686,8 +686,7 @@ WMFVideoMFTManager::InitInternal()
       (mUseHwAccel ? "Yes" : "No"));
 
   if (mUseHwAccel) {
-    hr = mDXVA2Manager->ConfigureForSize(mVideoInfo.ImageRect().width,
-                                         mVideoInfo.ImageRect().height);
+    hr = mDXVA2Manager->ConfigureForSize(outputType);
     NS_ENSURE_TRUE(SUCCEEDED(hr),
                    MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
                                RESULT_DETAIL("Fail to configure image size for "
@@ -1060,21 +1059,37 @@ WMFVideoMFTManager::Output(int64_t aStreamOffset,
 
     if (hr == MF_E_TRANSFORM_STREAM_CHANGE) {
       MOZ_ASSERT(!sample);
-      // Video stream output type change, probably geometric aperture change.
+      // Video stream output type change, probably geometric aperture change or
+      // pixel type.
       // We must reconfigure the decoder output type.
-      hr = mDecoder->SetDecoderOutputType(false /* check all attribute */,
-                                          nullptr,
-                                          nullptr);
+
+      // Attempt to find an appropriate OutputType, trying in order:
+      // if HW accelerated: NV12, P010, P016
+      // if SW: YV12
+      if (FAILED((hr = (mDecoder->FindDecoderOutputTypeWithSubtype(
+                    mUseHwAccel ? MFVideoFormat_NV12 : MFVideoFormat_YV12,
+                    false)))) &&
+          (!mUseHwAccel ||
+           (FAILED((hr = mDecoder->FindDecoderOutputTypeWithSubtype(
+                      MFVideoFormat_P010, false))) &&
+            FAILED((hr = mDecoder->FindDecoderOutputTypeWithSubtype(
+                      MFVideoFormat_P016, false)))))) {
+        LOG("No suitable output format found");
+        return hr;
+      }
+
+      RefPtr<IMFMediaType> outputType;
+      hr = mDecoder->GetOutputMediaType(outputType);
       NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
 
-      if (!mUseHwAccel) {
-        // The stride may have changed, recheck for it.
-        RefPtr<IMFMediaType> outputType;
-        hr = mDecoder->GetOutputMediaType(outputType);
+      if (mUseHwAccel) {
+        hr = mDXVA2Manager->ConfigureForSize(outputType);
         NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
+      } else {
+        // The stride may have changed, recheck for it.
         mYUVColorSpace = GetYUVColorSpace(outputType);
-        hr = GetDefaultStride(outputType, mVideoInfo.ImageRect().width,
-                              &mVideoStride);
+        hr = GetDefaultStride(
+          outputType, mVideoInfo.ImageRect().width, &mVideoStride);
         NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
 
         UINT32 width = 0, height = 0;

gfx/layers/D3D11ShareHandleImage.cpp

@@ -32,14 +32,18 @@ D3D11ShareHandleImage::D3D11ShareHandleImage(const gfx::IntSize& aSize,
 }
 
 bool
-D3D11ShareHandleImage::AllocateTexture(D3D11RecycleAllocator* aAllocator, ID3D11Device* aDevice)
+D3D11ShareHandleImage::AllocateTexture(D3D11RecycleAllocator* aAllocator,
+                                       ID3D11Device* aDevice,
+                                       bool aPreferNV12)
 {
   if (aAllocator) {
-    if (gfxPrefs::PDMWMFUseNV12Format() &&
+    if (aPreferNV12 && gfxPrefs::PDMWMFUseNV12Format() &&
         gfx::DeviceManagerDx::Get()->CanUseNV12()) {
-      mTextureClient = aAllocator->CreateOrRecycleClient(gfx::SurfaceFormat::NV12, mSize);
+      mTextureClient =
+        aAllocator->CreateOrRecycleClient(gfx::SurfaceFormat::NV12, mSize);
     } else {
-      mTextureClient = aAllocator->CreateOrRecycleClient(gfx::SurfaceFormat::B8G8R8A8, mSize);
+      mTextureClient =
+        aAllocator->CreateOrRecycleClient(gfx::SurfaceFormat::B8G8R8A8, mSize);
     }
     if (mTextureClient) {
       mTexture = static_cast<D3D11TextureData*>(mTextureClient->GetInternalData())->GetD3D11Texture();
@@ -87,7 +91,7 @@ D3D11ShareHandleImage::GetAsSourceSurface()
 
   HRESULT hr;
 
-  if (desc.Format == DXGI_FORMAT_NV12) {
+  if (desc.Format != DXGI_FORMAT_B8G8R8A8_UNORM) {
     nsAutoCString error;
     std::unique_ptr<DXVA2Manager> manager(DXVA2Manager::CreateD3D11DXVA(nullptr, error, device));
 

gfx/layers/D3D11ShareHandleImage.h

@@ -55,7 +55,9 @@ public:
                         const gfx::IntRect& aRect);
   virtual ~D3D11ShareHandleImage() {}
 
-  bool AllocateTexture(D3D11RecycleAllocator* aAllocator, ID3D11Device* aDevice);
+  bool AllocateTexture(D3D11RecycleAllocator* aAllocator,
+                       ID3D11Device* aDevice,
+                       bool aPreferNV12);
 
   gfx::IntSize GetSize() const override;
   already_AddRefed<gfx::SourceSurface> GetAsSourceSurface() override;