Bug 1423253 - Route MediaEncoder::Suspend/Resume through the graph thread for less drift. r=padenot

Because we sync video frames to audio time before encoding, we suspend and
resume on the audio thread because that's the only place we have acccess to
both clocks at the same time;
- A dispatch to the audio encoder puts the event in the right place in the
  audio encoder buffer.
- TimeStamp::Now() is the current video time, and since we capture it on the
  audio thread these are in sync. This timestamp needs drift compensation too -
  this happens in a later patch on this bug.

Differential Revision: https://phabricator.services.mozilla.com/D22907

--HG--
extra : moz-landing-system : lando

dom/media/MediaRecorder.cpp

@@ -592,7 +592,7 @@ class MediaRecorder::Session : public PrincipalChangeObserver<MediaStreamTrack>,
       return NS_ERROR_FAILURE;
     }
 
-    mEncoder->Suspend(TimeStamp::Now());
+    mEncoder->Suspend();
     NS_DispatchToMainThread(
         new DispatchEventRunnable(this, NS_LITERAL_STRING("pause")));
     return NS_OK;
@@ -606,7 +606,7 @@ class MediaRecorder::Session : public PrincipalChangeObserver<MediaStreamTrack>,
       return NS_ERROR_FAILURE;
     }
 
-    mEncoder->Resume(TimeStamp::Now());
+    mEncoder->Resume();
     NS_DispatchToMainThread(
         new DispatchEventRunnable(this, NS_LITERAL_STRING("resume")));
     return NS_OK;

dom/media/encoder/MediaEncoder.cpp

@@ -11,6 +11,7 @@
 #include "DriftCompensation.h"
 #include "GeckoProfiler.h"
 #include "MediaDecoder.h"
+#include "MediaStreamGraphImpl.h"
 #include "MediaStreamListener.h"
 #include "mozilla/dom/AudioNode.h"
 #include "mozilla/dom/AudioStreamTrack.h"
@@ -26,6 +27,7 @@
 #include "mozilla/Unused.h"
 #include "nsIPrincipal.h"
 #include "nsMimeTypes.h"
+#include "nsThreadUtils.h"
 #include "OggWriter.h"
 #include "OpusTrackEncoder.h"
 #include "TimeUnits.h"
@@ -434,36 +436,66 @@ MediaEncoder::MediaEncoder(TaskQueue* aEncoderThread,
 
 MediaEncoder::~MediaEncoder() { MOZ_ASSERT(mListeners.IsEmpty()); }
 
-void MediaEncoder::Suspend(TimeStamp aTime) {
-  auto& ae = mAudioEncoder;
-  auto& ve = mVideoEncoder;
-  nsresult rv = mEncoderThread->Dispatch(NewRunnableFrom([ae, ve, aTime]() {
-    if (ae) {
-      ae->Suspend(aTime);
-    }
-    if (ve) {
-      ve->Suspend(aTime);
-    }
-    return NS_OK;
-  }));
-  MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
-  Unused << rv;
+void MediaEncoder::RunOnGraph(already_AddRefed<Runnable> aRunnable) {
+  MediaStreamGraphImpl* graph;
+  if (mAudioTrack) {
+    graph = mAudioTrack->GraphImpl();
+  } else if (mVideoTrack) {
+    graph = mVideoTrack->GraphImpl();
+  } else if (mPipeStream) {
+    graph = mPipeStream->GraphImpl();
+  } else {
+    MOZ_CRASH("No graph");
+  }
+  class Message : public ControlMessage {
+   public:
+    explicit Message(already_AddRefed<Runnable> aRunnable)
+        : ControlMessage(nullptr), mRunnable(aRunnable) {}
+    void Run() override { mRunnable->Run(); }
+    const RefPtr<Runnable> mRunnable;
+  };
+  graph->AppendMessage(MakeUnique<Message>(std::move(aRunnable)));
 }
 
-void MediaEncoder::Resume(TimeStamp aTime) {
-  auto& ae = mAudioEncoder;
-  auto& ve = mVideoEncoder;
-  nsresult rv = mEncoderThread->Dispatch(NewRunnableFrom([ae, ve, aTime]() {
-    if (ae) {
-      ae->Resume(aTime);
-    }
-    if (ve) {
-      ve->Resume(aTime);
-    }
-    return NS_OK;
-  }));
-  MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
-  Unused << rv;
+void MediaEncoder::Suspend() {
+  RunOnGraph(NS_NewRunnableFunction(
+      "MediaEncoder::Suspend",
+      [thread = mEncoderThread, audio = mAudioEncoder, video = mVideoEncoder] {
+        if (audio) {
+          nsresult rv = thread->Dispatch(
+              NewRunnableMethod("AudioTrackEncoder::Suspend", audio,
+                                &AudioTrackEncoder::Suspend));
+          MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
+          Unused << rv;
+        }
+        if (video) {
+          nsresult rv = thread->Dispatch(NewRunnableMethod<TimeStamp>(
+              "VideoTrackEncoder::Suspend", video, &VideoTrackEncoder::Suspend,
+              TimeStamp::Now()));
+          MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
+          Unused << rv;
+        }
+      }));
+}
+
+void MediaEncoder::Resume() {
+  RunOnGraph(NS_NewRunnableFunction(
+      "MediaEncoder::Resume",
+      [thread = mEncoderThread, audio = mAudioEncoder, video = mVideoEncoder] {
+        if (audio) {
+          nsresult rv = thread->Dispatch(NewRunnableMethod(
+              "AudioTrackEncoder::Resume", audio, &AudioTrackEncoder::Resume));
+          MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
+          Unused << rv;
+        }
+        if (video) {
+          nsresult rv = thread->Dispatch(NewRunnableMethod<TimeStamp>(
+              "VideoTrackEncoder::Resume", video, &VideoTrackEncoder::Resume,
+              TimeStamp::Now()));
+          MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
+          Unused << rv;
+        }
+      }));
 }
 
 void MediaEncoder::ConnectAudioNode(AudioNode* aNode, uint32_t aOutput) {

dom/media/encoder/MediaEncoder.h

@@ -19,6 +19,7 @@
 namespace mozilla {
 
 class DriftCompensator;
+class Runnable;
 class TaskQueue;
 
 namespace dom {
@@ -115,13 +116,15 @@ class MediaEncoder {
                VideoTrackEncoder* aVideoEncoder, TrackRate aTrackRate,
                const nsAString& aMIMEType);
 
-  /* Note - called from control code, not on MSG threads. */
-  void Suspend(TimeStamp aTime);
+  /**
+   * Called on main thread from MediaRecorder::Pause.
+   */
+  void Suspend();
 
   /**
-   * Note - called from control code, not on MSG threads.
-   * Calculates time spent paused in order to offset frames. */
-  void Resume(TimeStamp aTime);
+   * Called on main thread from MediaRecorder::Resume.
+   */
+  void Resume();
 
   /**
    * Stops the current encoding, and disconnects the input tracks.
@@ -231,6 +234,12 @@ class MediaEncoder {
   ~MediaEncoder();
 
  private:
+  /**
+   * Takes a regular runnable and dispatches it to the graph wrapped in a
+   * ControlMessage.
+   */
+  void RunOnGraph(already_AddRefed<Runnable> aRunnable);
+
   /**
    * Shuts down the MediaEncoder and cleans up track encoders.
    * Listeners will be notified of the shutdown unless we were Cancel()ed first.

dom/media/encoder/TrackEncoder.cpp

@@ -102,7 +102,7 @@ void TrackEncoder::SetWorkerThread(AbstractThread* aWorkerThread) {
   mWorkerThread = aWorkerThread;
 }
 
-void AudioTrackEncoder::Suspend(TimeStamp) {
+void AudioTrackEncoder::Suspend() {
   MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
   TRACK_LOG(LogLevel::Info, ("[AudioTrackEncoder %p]: Suspend(), was %s", this,
                              mSuspended ? "suspended" : "live"));
@@ -114,7 +114,7 @@ void AudioTrackEncoder::Suspend(TimeStamp) {
   mSuspended = true;
 }
 
-void AudioTrackEncoder::Resume(TimeStamp) {
+void AudioTrackEncoder::Resume() {
   MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
   TRACK_LOG(LogLevel::Info, ("[AudioTrackEncoder %p]: Resume(), was %s", this,
                              mSuspended ? "suspended" : "live"));

dom/media/encoder/TrackEncoder.h

@@ -61,10 +61,6 @@ class TrackEncoder {
  public:
   explicit TrackEncoder(TrackRate aTrackRate);
 
-  virtual void Suspend(TimeStamp aTime) = 0;
-
-  virtual void Resume(TimeStamp aTime) = 0;
-
   /**
    * Called by MediaEncoder to cancel the encoding.
    */
@@ -200,13 +196,13 @@ class AudioTrackEncoder : public TrackEncoder {
    * Suspends encoding from now, i.e., all future audio data received through
    * AppendAudioSegment() until the next Resume() will be dropped.
    */
-  void Suspend(TimeStamp aTime) override;
+  void Suspend();
 
   /**
    * Resumes encoding starting now, i.e., data from the next
    * AppendAudioSegment() will get encoded.
    */
-  void Resume(TimeStamp aTime) override;
+  void Resume();
 
   /**
    * Appends and consumes track data from aSegment.
@@ -340,12 +336,12 @@ class VideoTrackEncoder : public TrackEncoder {
    * Suspends encoding from aTime, i.e., all video frame with a timestamp
    * between aTime and the timestamp of the next Resume() will be dropped.
    */
-  void Suspend(TimeStamp aTime) override;
+  void Suspend(TimeStamp aTime);
 
   /**
    * Resumes encoding starting at aTime.
    */
-  void Resume(TimeStamp aTime) override;
+  void Resume(TimeStamp aTime);
 
   /**
    * Appends source video frames to mIncomingBuffer. We only append the source

dom/media/encoder/moz.build

@@ -33,8 +33,10 @@ DEFINES['TRACING'] = True
 
 FINAL_LIBRARY = 'xul'
 
-# These includes are from Android JB, for use of MediaCodec.
-LOCAL_INCLUDES += ['/ipc/chromium/src']
+LOCAL_INCLUDES += [
+    '/dom/media',
+    '/ipc/chromium/src',
+]
 
 include('/ipc/chromium/chromium-config.mozbuild')