Bug 1509548 - Make MediaStreamGraph pull data per track instead of per stream. r=padenot

Differential Revision: https://phabricator.services.mozilla.com/D13086

--HG--
extra : moz-landing-system : lando

dom/media/CanvasCaptureMediaStream.cpp

@@ -115,7 +115,7 @@ OutputStreamDriver::OutputStreamDriver(SourceMediaStream* aSourceStream,
   MOZ_ASSERT(mSourceStream);
   mSourceStream->AddTrack(aTrackId, new VideoSegment());
   mSourceStream->AddTrackListener(mTrackListener, aTrackId);
-  mSourceStream->SetPullEnabled(true);
+  mSourceStream->SetPullingEnabled(aTrackId, true);
 
   // All CanvasCaptureMediaStreams shall at least get one frame.
   mFrameCaptureRequested = true;

dom/media/MediaManager.cpp

@@ -4207,8 +4207,6 @@ RefPtr<SourceListener::InitPromise> SourceListener::InitializeAsync() {
           return InitPromise::CreateAndResolve(true, __func__);
         }
 
-        mStream->SetPullEnabled(true);
-
         for (DeviceState* state :
              {mAudioDeviceState.get(), mVideoDeviceState.get()}) {
           if (!state) {
@@ -4221,6 +4219,15 @@ RefPtr<SourceListener::InitPromise> SourceListener::InitializeAsync() {
           state->mDeviceEnabled = true;
           state->mTrackEnabled = true;
           state->mTrackEnabledTime = TimeStamp::Now();
+
+          if (state->mDevice->GetMediaSource() !=
+              MediaSourceEnum::AudioCapture) {
+            // For AudioCapture mStream is a dummy stream, so we don't try to
+            // enable pulling - there won't be a track to enable it for.
+            mStream->SetPullingEnabled(
+                state == mAudioDeviceState.get() ? kAudioTrack : kVideoTrack,
+                true);
+          }
         }
         return InitPromise::CreateAndResolve(true, __func__);
       },
@@ -4295,11 +4302,12 @@ void SourceListener::Remove() {
     // We disable pulling before removing so we don't risk having live tracks
     // without a listener attached - that wouldn't produce data and would be
     // illegal to the graph.
-    mStream->SetPullEnabled(false);
     if (mAudioDeviceState) {
+      mStream->SetPullingEnabled(kAudioTrack, false);
       mStream->RemoveTrackListener(mAudioDeviceState->mListener, kAudioTrack);
     }
     if (mVideoDeviceState) {
+      mStream->SetPullingEnabled(kVideoTrack, false);
       mStream->RemoveTrackListener(mVideoDeviceState->mListener, kVideoTrack);
     }
   }

dom/media/MediaStreamGraph.cpp

@@ -1221,12 +1221,21 @@ void MediaStreamGraphImpl::UpdateGraph(GraphTime aEndBlockingDecisions) {
     } else {
       stream->mStartBlocking = WillUnderrun(stream, aEndBlockingDecisions);
 
-      SourceMediaStream* s = stream->AsSourceStream();
-      if (s && s->mPullEnabled) {
+#ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED
+      if (SourceMediaStream* s = stream->AsSourceStream()) {
         for (StreamTracks::TrackIter i(s->mTracks); !i.IsEnded(); i.Next()) {
           if (i->IsEnded()) {
             continue;
           }
+          SourceMediaStream::TrackData* data;
+          {
+            MutexAutoLock lock(s->mMutex);
+            data = s->FindDataForTrack(i->GetID());
+          }
+          MOZ_ASSERT(data);
+          if (!data->mPullingEnabled) {
+            continue;
+          }
           if (i->GetEnd() <
               stream->GraphTimeToStreamTime(aEndBlockingDecisions)) {
             LOG(LogLevel::Error,
@@ -1246,6 +1255,7 @@ void MediaStreamGraphImpl::UpdateGraph(GraphTime aEndBlockingDecisions) {
           }
         }
       }
+#endif /* MOZ_DIAGNOSTIC_ASSERT_ENABLED */
     }
   }
 
@@ -2459,7 +2469,6 @@ void MediaStream::AddMainThreadListener(
 SourceMediaStream::SourceMediaStream()
     : MediaStream(),
       mMutex("mozilla::media::SourceMediaStream"),
-      mPullEnabled(false),
       mFinishPending(false) {}
 
 nsresult SourceMediaStream::OpenAudioInput(CubebUtils::AudioDeviceID aID,
@@ -2496,25 +2505,39 @@ void SourceMediaStream::DestroyImpl() {
   MediaStream::DestroyImpl();
 }
 
-void SourceMediaStream::SetPullEnabled(bool aEnabled) {
+void SourceMediaStream::SetPullingEnabled(TrackID aTrackID, bool aEnabled) {
   class Message : public ControlMessage {
    public:
-    Message(SourceMediaStream* aStream, bool aEnabled)
-        : ControlMessage(nullptr), mStream(aStream), mEnabled(aEnabled) {}
+    Message(SourceMediaStream* aStream, TrackID aTrackID, bool aEnabled)
+        : ControlMessage(nullptr),
+          mStream(aStream),
+          mTrackID(aTrackID),
+          mEnabled(aEnabled) {}
     void Run() override {
       MutexAutoLock lock(mStream->mMutex);
-      mStream->mPullEnabled = mEnabled;
+      TrackData* data = mStream->FindDataForTrack(mTrackID);
+      if (!data) {
+        // We can't enable pulling for a track that was never added. We ignore
+        // this if we're disabling pulling, since shutdown sequences are
+        // complex. If there's truly an issue we'll have issues enabling anyway.
+        MOZ_ASSERT_IF(mEnabled,
+                      mStream->mTracks.FindTrack(mTrackID) &&
+                          mStream->mTracks.FindTrack(mTrackID)->IsEnded());
+        return;
+      }
+      data->mPullingEnabled = mEnabled;
     }
     SourceMediaStream* mStream;
+    TrackID mTrackID;
     bool mEnabled;
   };
-  GraphImpl()->AppendMessage(MakeUnique<Message>(this, aEnabled));
+  GraphImpl()->AppendMessage(MakeUnique<Message>(this, aTrackID, aEnabled));
 }
 
 bool SourceMediaStream::PullNewData(GraphTime aDesiredUpToTime) {
   TRACE_AUDIO_CALLBACK_COMMENT("SourceMediaStream %p", this);
   MutexAutoLock lock(mMutex);
-  if (!mPullEnabled || mFinished) {
+  if (mFinished) {
     return false;
   }
   // Compute how much stream time we'll need assuming we don't block
@@ -2526,6 +2549,9 @@ bool SourceMediaStream::PullNewData(GraphTime aDesiredUpToTime) {
        this, GraphImpl()->MediaTimeToSeconds(t),
        GraphImpl()->MediaTimeToSeconds(current)));
   for (const TrackData& track : mUpdateTracks) {
+    if (!track.mPullingEnabled) {
+      continue;
+    }
     if (track.mCommands & TrackEventCommand::TRACK_EVENT_ENDED) {
       continue;
     }
@@ -2623,6 +2649,7 @@ void SourceMediaStream::AddTrackInternal(TrackID aID, TrackRate aRate,
   data->mEndOfFlushedData = 0;
   data->mCommands = TRACK_CREATE;
   data->mData = aSegment;
+  data->mPullingEnabled = false;
   ResampleAudioToGraphSampleRate(data, aSegment);
   if (!(aFlags & ADDTRACK_QUEUED) && GraphImpl()) {
     GraphImpl()->EnsureNextIteration();

dom/media/MediaStreamGraph.h

@@ -658,13 +658,14 @@ class SourceMediaStream : public MediaStream {
   // Main thread only
 
   /**
-   * Enable or disable pulling. When pulling is enabled, NotifyPull
-   * gets called on MediaStream/TrackListeners for this stream during the
-   * MediaStreamGraph control loop. Pulling is initially disabled.
-   * Due to unavoidable race conditions, after a call to SetPullEnabled(false)
+   * Enable or disable pulling for a specific track.
+   * When pulling is enabled, NotifyPull gets called on the corresponding
+   * MediaStreamTrackListeners for this stream during the MediaStreamGraph
+   * control loop. Pulling is initially disabled for all tracks. Due to
+   * unavoidable race conditions, after a call to SetPullingEnabled(false)
    * it is still possible for a NotifyPull to occur.
    */
-  void SetPullEnabled(bool aEnabled);
+  void SetPullingEnabled(TrackID aTrackID, bool aEnabled);
 
   // Users of audio inputs go through the stream so it can track when the
   // last stream referencing an input goes away, so it can close the cubeb
@@ -809,6 +810,8 @@ class SourceMediaStream : public MediaStream {
     // Each time the track updates are flushed to the media graph thread,
     // this is cleared.
     uint32_t mCommands;
+    // True if the producer of this track is having data pulled by the graph.
+    bool mPullingEnabled;
   };
 
   bool NeedsMixing();
@@ -867,7 +870,6 @@ class SourceMediaStream : public MediaStream {
   nsTArray<TrackData> mUpdateTracks;
   nsTArray<TrackData> mPendingTracks;
   nsTArray<TrackBound<DirectMediaStreamTrackListener>> mDirectTrackListeners;
-  bool mPullEnabled;
   bool mFinishPending;
 };
 

media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp

@@ -1680,7 +1680,7 @@ public:
   {
     if (!mListening) {
       mListening = true;
-      mSource->SetPullEnabled(true);
+      mSource->SetPullingEnabled(mTrackId, true);
       mMaybeTrackNeedsUnmute = true;
     }
   }
@@ -1689,7 +1689,7 @@ public:
   {
     if (mListening) {
       mListening = false;
-      mSource->SetPullEnabled(false);
+      mSource->SetPullingEnabled(mTrackId, false);
     }
   }