Bug 1741244 - p1: support software MFT video encoders. r=alwu,media-playback-reviewers

Differential Revision: https://phabricator.services.mozilla.com/D135766
2022-02-23 22:44:55 +00:00 · 2022-02-23 22:44:55 +00:00 · 7dbc14300f
--- a/dom/media/platforms/wmf/MFTEncoder.cpp
+++ b/dom/media/platforms/wmf/MFTEncoder.cpp
@ -86,30 +86,43 @@ static const char* CodecStr(const GUID& aGUID) {
  }
 }

-static UINT32 EnumHW(const GUID& aSubtype, IMFActivate**& aActivates) {
-  if (IsWin32kLockedDown()) {
-    // Some HW encoders use system calls and crash when locked down.
-    // TODO: move HW encoding to RDD.
-    MFT_ENC_SLOGD("Don't use HW encoder when win32k locked down.");
-    return 0;
-  }
-
+static UINT32 EnumEncoders(const GUID& aSubtype, IMFActivate**& aActivates) {
  UINT32 num = 0;
  MFT_REGISTER_TYPE_INFO inType = {.guidMajorType = MFMediaType_Video,
                                   .guidSubtype = MFVideoFormat_NV12};
  MFT_REGISTER_TYPE_INFO outType = {.guidMajorType = MFMediaType_Video,
                                    .guidSubtype = aSubtype};
-  HRESULT hr =
-      wmf::MFTEnumEx(MFT_CATEGORY_VIDEO_ENCODER,
-                     MFT_ENUM_FLAG_HARDWARE | MFT_ENUM_FLAG_SORTANDFILTER,
-                     &inType, &outType, &aActivates, &num);
+  HRESULT hr = S_OK;
+  if (IsWin32kLockedDown()) {
+    // Some HW encoders use DXGI API and crash when locked down.
+    // TODO: move HW encoding out of content process (bug 1754531).
+    MFT_ENC_SLOGD("Don't use HW encoder when win32k locked down.");
+  } else {
+    hr = wmf::MFTEnumEx(MFT_CATEGORY_VIDEO_ENCODER,
+                        MFT_ENUM_FLAG_HARDWARE | MFT_ENUM_FLAG_SORTANDFILTER,
+                        &inType, &outType, &aActivates, &num);
+  }
  if (FAILED(hr)) {
    MFT_ENC_SLOGE("enumerate HW encoder for %s: error=%s", CodecStr(aSubtype),
                  ErrorStr(hr));
    return 0;
  }
+
+  if (num == 0 && StaticPrefs::media_webrtc_platformencoder_sw_mft()) {
+    // Try software MFTs.
+    hr = wmf::MFTEnumEx(MFT_CATEGORY_VIDEO_ENCODER,
+                        MFT_ENUM_FLAG_SYNCMFT | MFT_ENUM_FLAG_ASYNCMFT |
+                            MFT_ENUM_FLAG_SORTANDFILTER,
+                        &inType, &outType, &aActivates, &num);
+    if (FAILED(hr)) {
+      MFT_ENC_SLOGE("enumerate SW encoder for %s: error=%s", CodecStr(aSubtype),
+                    ErrorStr(hr));
+      return 0;
+    }
+  }
+
  if (num == 0) {
-    MFT_ENC_SLOGD("cannot find HW encoder for %s", CodecStr(aSubtype));
+    MFT_ENC_SLOGD("cannot find encoder for %s", CodecStr(aSubtype));
  }
  return num;
 }
@ -134,10 +147,10 @@ static HRESULT GetFriendlyName(IMFActivate* aAttributes, nsCString& aName) {
  return S_OK;
 }

-static void PopulateHWEncoderInfo(const GUID& aSubtype,
-                                  nsTArray<MFTEncoder::Info>& aInfos) {
+static void PopulateEncoderInfo(const GUID& aSubtype,
+                                nsTArray<MFTEncoder::Info>& aInfos) {
  IMFActivate** activates = nullptr;
-  UINT32 num = EnumHW(aSubtype, activates);
+  UINT32 num = EnumEncoders(aSubtype, activates);
  for (UINT32 i = 0; i < num; ++i) {
    MFTEncoder::Info info = {.mSubtype = aSubtype};
    GetFriendlyName(activates[i], info.mName);
@ -175,9 +188,9 @@ nsTArray<MFTEncoder::Info> MFTEncoder::Enumerate() {
    return infos;
  }

-  PopulateHWEncoderInfo(MFVideoFormat_H264, infos);
-  PopulateHWEncoderInfo(MFVideoFormat_VP90, infos);
-  PopulateHWEncoderInfo(MFVideoFormat_VP80, infos);
+  PopulateEncoderInfo(MFVideoFormat_H264, infos);
+  PopulateEncoderInfo(MFVideoFormat_VP90, infos);
+  PopulateEncoderInfo(MFVideoFormat_VP80, infos);

  wmf::MFShutdown();
  return infos;
@ -195,7 +208,7 @@ already_AddRefed<IMFActivate> MFTEncoder::CreateFactory(const GUID& aSubtype) {
  }

  IMFActivate** activates = nullptr;
-  UINT32 num = EnumHW(aSubtype, activates);
+  UINT32 num = EnumEncoders(aSubtype, activates);
  if (num == 0) {
    return nullptr;
  }
@ -248,7 +261,6 @@ MFTEncoder::Destroy() {
    return S_OK;
  }

-  mEventSource = nullptr;
  mEncoder = nullptr;
  mConfig = nullptr;
  // Release MFT resources via activation object.
@ -263,10 +275,10 @@ MFTEncoder::SetMediaTypes(IMFMediaType* aInputType, IMFMediaType* aOutputType) {
  MOZ_ASSERT(mscom::IsCurrentThreadMTA());
  MOZ_ASSERT(aInputType && aOutputType);

-  HRESULT hr = EnableAsync();
-  NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
+  AsyncMFTResult asyncMFT = AttemptEnableAsync();
+  NS_ENSURE_TRUE(asyncMFT.isOk(), asyncMFT.unwrapErr());

-  hr = GetStreamIDs();
+  HRESULT hr = GetStreamIDs();
  NS_ENSURE_TRUE(SUCCEEDED(hr), hr);

  // Always set encoder output type before input.
@ -287,40 +299,45 @@ MFTEncoder::SetMediaTypes(IMFMediaType* aInputType, IMFMediaType* aOutputType) {
  mOutputStreamProvidesSample =
      IsFlagSet(mOutputStreamInfo.dwFlags, MFT_OUTPUT_STREAM_PROVIDES_SAMPLES);

-  hr = SendMFTMessage(MFT_MESSAGE_COMMAND_FLUSH, 0);
-  NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
-
  hr = SendMFTMessage(MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, 0);
  NS_ENSURE_TRUE(SUCCEEDED(hr), hr);

  hr = SendMFTMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0);
  NS_ENSURE_TRUE(SUCCEEDED(hr), hr);

-  RefPtr<IMFMediaEventGenerator> source;
-  hr = mEncoder->QueryInterface(IID_PPV_ARGS(
-      static_cast<IMFMediaEventGenerator**>(getter_AddRefs(source))));
-  NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
-  mEventSource = std::move(source);
+  if (asyncMFT.unwrap()) {
+    RefPtr<IMFMediaEventGenerator> source;
+    hr = mEncoder->QueryInterface(IID_PPV_ARGS(
+        static_cast<IMFMediaEventGenerator**>(getter_AddRefs(source))));
+    NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
+    mEventSource.SetAsyncEventGenerator(source.forget());
+  } else {
+    mEventSource.InitSyncMFTEventQueue();
+  }
+
  mNumNeedInput = 0;
  return S_OK;
 }

 // Async MFT won't work without unlocking. See
 // https://docs.microsoft.com/en-us/windows/win32/medfound/asynchronous-mfts#unlocking-asynchronous-mfts
-HRESULT MFTEncoder::EnableAsync() {
+MFTEncoder::AsyncMFTResult MFTEncoder::AttemptEnableAsync() {
  IMFAttributes* pAttributes = nullptr;
  HRESULT hr = mEncoder->GetAttributes(&pAttributes);
-  NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
+  if (FAILED(hr)) {
+    return AsyncMFTResult(hr);
+  }

-  UINT32 async = MFGetAttributeUINT32(pAttributes, MF_TRANSFORM_ASYNC, FALSE);
-  if (async == TRUE) {
+  bool async =
+      MFGetAttributeUINT32(pAttributes, MF_TRANSFORM_ASYNC, FALSE) == TRUE;
+  if (async) {
    hr = pAttributes->SetUINT32(MF_TRANSFORM_ASYNC_UNLOCK, TRUE);
  } else {
-    hr = E_NOTIMPL;
+    hr = S_OK;
  }
  pAttributes->Release();

-  return hr;
+  return SUCCEEDED(hr) ? AsyncMFTResult(async) : AsyncMFTResult(hr);
 }

 HRESULT MFTEncoder::GetStreamIDs() {
@ -467,6 +484,12 @@ MFTEncoder::PushInput(RefPtr<IMFSample>&& aInput) {
  MOZ_ASSERT(aInput);

  mPendingInputs.Push(aInput.forget());
+  if (mEventSource.IsSync() && mNumNeedInput == 0) {
+    // To step 2 in
+    // https://docs.microsoft.com/en-us/windows/win32/medfound/basic-mft-processing-model#process-data
+    mNumNeedInput++;
+  }
+
  HRESULT hr = ProcessInput();
  NS_ENSURE_TRUE(SUCCEEDED(hr), hr);

@ -486,33 +509,44 @@ HRESULT MFTEncoder::ProcessInput() {
  NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
  --mNumNeedInput;

-  return S_OK;
+  if (!mEventSource.IsSync()) {
+    return S_OK;
+  }
+  // For sync MFT: Step 3 in
+  // https://docs.microsoft.com/en-us/windows/win32/medfound/basic-mft-processing-model#process-data
+  DWORD flags = 0;
+  hr = mEncoder->GetOutputStatus(&flags);
+  MediaEventType evType = MEUnknown;
+  switch (hr) {
+    case S_OK:
+      evType = flags == MFT_OUTPUT_STATUS_SAMPLE_READY
+                   ? METransformHaveOutput  // To step 4: ProcessOutput().
+                   : METransformNeedInput;  // To step 2: ProcessInput().
+      break;
+    case E_NOTIMPL:
+      evType = METransformHaveOutput;  // To step 4: ProcessOutput().
+      break;
+    default:
+      MOZ_MAKE_COMPILER_ASSUME_IS_UNREACHABLE("undefined output status");
+      return hr;
+  }
+  return mEventSource.QueueSyncMFTEvent(evType);
 }

 HRESULT MFTEncoder::ProcessEvents() {
  MOZ_ASSERT(mscom::IsCurrentThreadMTA());
  MOZ_ASSERT(mEncoder);
-  MOZ_ASSERT(mEventSource, "no event generator");

  HRESULT hr = E_FAIL;
  while (true) {
-    RefPtr<IMFMediaEvent> event;
-    hr = mEventSource->GetEvent(MF_EVENT_FLAG_NO_WAIT, getter_AddRefs(event));
-    switch (hr) {
-      case S_OK:
-        break;
-      case MF_E_NO_EVENTS_AVAILABLE:
-        return S_OK;
-      case MF_E_MULTIPLE_SUBSCRIBERS:
-      default:
-        MFT_ENC_LOGE("failed to get event: %s", ErrorStr(hr));
-        return hr;
+    Event event = mEventSource.GetEvent();
+    if (event.isErr()) {
+      hr = event.unwrapErr();
+      break;
    }

-    MediaEventType type = MEUnknown;
-    HRESULT hr = event->GetType(&type);
-    NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
-    switch (type) {
+    MediaEventType evType = event.unwrap();
+    switch (evType) {
      case METransformNeedInput:
        ++mNumNeedInput;
        hr = ProcessInput();
@ -520,16 +554,24 @@ HRESULT MFTEncoder::ProcessEvents() {
        break;
      case METransformHaveOutput:
        hr = ProcessOutput();
+        NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
        break;
      case METransformDrainComplete:
        mDrainState = DrainState::DRAINED;
        break;
      default:
-        MFT_ENC_LOGE("event: error=%s", ErrorStr(hr));
+        MFT_ENC_LOGE("unsupported event: %x", evType);
    }
  }

-  return hr;
+  switch (hr) {
+    case MF_E_NO_EVENTS_AVAILABLE:
+      return S_OK;
+    case MF_E_MULTIPLE_SUBSCRIBERS:
+    default:
+      MFT_ENC_LOGE("failed to get event: %s", ErrorStr(hr));
+      return hr;
+  }
 }

 HRESULT MFTEncoder::ProcessOutput() {
@ -566,6 +608,17 @@ HRESULT MFTEncoder::ProcessOutput() {
    }
    return MF_E_TRANSFORM_STREAM_CHANGE;
  }
+
+  // Step 8 in
+  // https://docs.microsoft.com/en-us/windows/win32/medfound/basic-mft-processing-model#process-data
+  if (hr == MF_E_TRANSFORM_NEED_MORE_INPUT) {
+    MOZ_ASSERT(mEventSource.IsSync());
+    MOZ_ASSERT(mDrainState == DrainState::DRAINING);
+
+    mEventSource.QueueSyncMFTEvent(METransformDrainComplete);
+    return S_OK;
+  }
+
  NS_ENSURE_TRUE(SUCCEEDED(hr), hr);

  mOutputs.AppendElement(output.pSample);
@ -593,6 +646,11 @@ HRESULT MFTEncoder::Drain(nsTArray<RefPtr<IMFSample>>& aOutput) {
    case DrainState::DRAINABLE:
      // Exhaust pending inputs.
      while (mPendingInputs.GetSize() > 0) {
+        if (mEventSource.IsSync()) {
+          // Step 5 in
+          // https://docs.microsoft.com/en-us/windows/win32/medfound/basic-mft-processing-model#process-data
+          mEventSource.QueueSyncMFTEvent(METransformNeedInput);
+        }
        HRESULT hr = ProcessEvents();
        NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
      }
@ -602,6 +660,11 @@ HRESULT MFTEncoder::Drain(nsTArray<RefPtr<IMFSample>>& aOutput) {
    case DrainState::DRAINING:
      // Collect remaining outputs.
      while (mOutputs.Length() == 0 && mDrainState != DrainState::DRAINED) {
+        if (mEventSource.IsSync()) {
+          // Step 8 in
+          // https://docs.microsoft.com/en-us/windows/win32/medfound/basic-mft-processing-model#process-data
+          mEventSource.QueueSyncMFTEvent(METransformHaveOutput);
+        }
        HRESULT hr = ProcessEvents();
        NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
      }
@ -634,6 +697,52 @@ HRESULT MFTEncoder::GetMPEGSequenceHeader(nsTArray<UINT8>& aHeader) {
  return hr;
 }

+MFTEncoder::Event MFTEncoder::EventSource::GetEvent() {
+  if (IsSync()) {
+    return GetSyncMFTEvent();
+  }
+
+  RefPtr<IMFMediaEvent> event;
+  HRESULT hr = mImpl.as<RefPtr<IMFMediaEventGenerator>>()->GetEvent(
+      MF_EVENT_FLAG_NO_WAIT, getter_AddRefs(event));
+  MediaEventType type = MEUnknown;
+  if (SUCCEEDED(hr)) {
+    hr = event->GetType(&type);
+  }
+  return SUCCEEDED(hr) ? Event{type} : Event{hr};
+}
+
+HRESULT MFTEncoder::EventSource::QueueSyncMFTEvent(MediaEventType aEventType) {
+  MOZ_ASSERT(IsSync());
+  MOZ_ASSERT(IsOnCurrentThread());
+
+  auto q = mImpl.as<UniquePtr<EventQueue>>().get();
+  q->push(aEventType);
+  return S_OK;
+}
+
+MFTEncoder::Event MFTEncoder::EventSource::GetSyncMFTEvent() {
+  MOZ_ASSERT(IsOnCurrentThread());
+
+  auto q = mImpl.as<UniquePtr<EventQueue>>().get();
+  if (q->empty()) {
+    return Event{MF_E_NO_EVENTS_AVAILABLE};
+  }
+
+  MediaEventType type = q->front();
+  q->pop();
+  return Event{type};
+}
+
+#ifdef DEBUG
+bool MFTEncoder::EventSource::IsOnCurrentThread() {
+  if (!mThread) {
+    mThread = GetCurrentSerialEventTarget();
+  }
+  return mThread->IsOnCurrentThread();
+}
+#endif
+
 }  // namespace mozilla

 #undef MFT_ENC_SLOGE
--- a/dom/media/platforms/wmf/MFTEncoder.h
+++ b/dom/media/platforms/wmf/MFTEncoder.h
@ -8,7 +8,9 @@
 #  define MFTEncoder_h_

 #  include <functional>
+#  include <queue>
 #  include "mozilla/RefPtr.h"
+#  include "mozilla/ResultVariant.h"
 #  include "nsISupportsImpl.h"
 #  include "nsDeque.h"
 #  include "WMF.h"
@ -40,6 +42,55 @@ class MFTEncoder final {
  };

 private:
+  // Abstractions to support sync MFTs using the same logic for async MFTs.
+  // When the MFT is async and a real event generator is available, simply
+  // forward the calls. For sync MFTs, use the synchronous processing model
+  // described in
+  // https://docs.microsoft.com/en-us/windows/win32/medfound/basic-mft-processing-model#process-data
+  // to generate events of the asynchronous processing model.
+  using Event = Result<MediaEventType, HRESULT>;
+  using EventQueue = std::queue<MediaEventType>;
+  class EventSource final {
+   public:
+    EventSource() : mImpl(Nothing{}) {}
+
+    void SetAsyncEventGenerator(
+        already_AddRefed<IMFMediaEventGenerator>&& aAsyncEventGenerator) {
+      MOZ_ASSERT(mImpl.is<Nothing>());
+      mImpl.emplace<RefPtr<IMFMediaEventGenerator>>(aAsyncEventGenerator);
+    }
+
+    void InitSyncMFTEventQueue() {
+      MOZ_ASSERT(mImpl.is<Nothing>());
+      mImpl.emplace<UniquePtr<EventQueue>>(MakeUnique<EventQueue>());
+    }
+
+    bool IsSync() const { return mImpl.is<UniquePtr<EventQueue>>(); }
+
+    Event GetEvent();
+    // Push an event when sync MFT is used.
+    HRESULT QueueSyncMFTEvent(MediaEventType aEventType);
+
+   private:
+    // Pop an event from the queue when sync MFT is used.
+    Event GetSyncMFTEvent();
+
+    Variant<
+        // Uninitialized.
+        Nothing,
+        // For async MFT events. See
+        // https://docs.microsoft.com/en-us/windows/win32/medfound/asynchronous-mfts#events
+        RefPtr<IMFMediaEventGenerator>,
+        // Event queue for a sync MFT. Storing EventQueue directly breaks the
+        // code so a pointer is introduced.
+        UniquePtr<EventQueue>>
+        mImpl;
+#  ifdef DEBUG
+    bool IsOnCurrentThread();
+    nsCOMPtr<nsISerialEventTarget> mThread;
+#  endif
+  };
+
  ~MFTEncoder() { Destroy(); };

  static nsTArray<Info>& Infos();
@ -47,7 +98,10 @@ class MFTEncoder final {
  static Maybe<Info> GetInfo(const GUID& aSubtype);

  already_AddRefed<IMFActivate> CreateFactory(const GUID& aSubtype);
-  HRESULT EnableAsync();
+  // Return true when successfully enabled, false for MFT that doesn't support
+  // async processing model, and error otherwise.
+  using AsyncMFTResult = Result<bool, HRESULT>;
+  AsyncMFTResult AttemptEnableAsync();
  HRESULT GetStreamIDs();
  GUID MatchInputSubtype(IMFMediaType* aInputType);
  HRESULT SendMFTMessage(MFT_MESSAGE_TYPE aMsg, ULONG_PTR aData);
@ -63,9 +117,6 @@ class MFTEncoder final {
  // For encoder configuration. See
  // https://docs.microsoft.com/en-us/windows/win32/directshow/encoder-api
  RefPtr<ICodecAPI> mConfig;
-  // For async MFT events. See
-  // https://docs.microsoft.com/en-us/windows/win32/medfound/asynchronous-mfts#events
-  RefPtr<IMFMediaEventGenerator> mEventSource;

  DWORD mInputStreamID;
  DWORD mOutputStreamID;
@ -79,6 +130,8 @@ class MFTEncoder final {

  nsRefPtrDeque<IMFSample> mPendingInputs;
  nsTArray<RefPtr<IMFSample>> mOutputs;
+
+  EventSource mEventSource;
 };

 }  // namespace mozilla
--- a/modules/libpref/init/StaticPrefList.yaml
+++ b/modules/libpref/init/StaticPrefList.yaml
@ -9470,6 +9470,11 @@
 #endif
  mirror: always

+- name: media.webrtc.platformencoder.sw_mft
+  type: RelaxedAtomicBool
+  value: @IS_EARLY_BETA_OR_EARLIER@
+  mirror: always
+
 - name: media.block-autoplay-until-in-foreground
  type: bool
 #if !defined(MOZ_WIDGET_ANDROID)