From f0bf48464f49c4eb6c9927690e3eb37d56bf4624 Mon Sep 17 00:00:00 2001 From: Andreas Pehrson Date: Thu, 29 Nov 2018 17:37:42 +0000 Subject: [PATCH] Bug 1509548 - Make MediaStreamGraph pull data per track instead of per stream. r=padenot Differential Revision: https://phabricator.services.mozilla.com/D13086 --HG-- extra : moz-landing-system : lando --- dom/media/CanvasCaptureMediaStream.cpp | 2 +- dom/media/MediaManager.cpp | 14 ++++-- dom/media/MediaStreamGraph.cpp | 45 +++++++++++++++---- dom/media/MediaStreamGraph.h | 14 +++--- .../src/mediapipeline/MediaPipeline.cpp | 4 +- 5 files changed, 58 insertions(+), 21 deletions(-) diff --git a/dom/media/CanvasCaptureMediaStream.cpp b/dom/media/CanvasCaptureMediaStream.cpp index d6867390d20a..a4179ac8ad61 100644 --- a/dom/media/CanvasCaptureMediaStream.cpp +++ b/dom/media/CanvasCaptureMediaStream.cpp @@ -115,7 +115,7 @@ OutputStreamDriver::OutputStreamDriver(SourceMediaStream* aSourceStream, MOZ_ASSERT(mSourceStream); mSourceStream->AddTrack(aTrackId, new VideoSegment()); mSourceStream->AddTrackListener(mTrackListener, aTrackId); - mSourceStream->SetPullEnabled(true); + mSourceStream->SetPullingEnabled(aTrackId, true); // All CanvasCaptureMediaStreams shall at least get one frame. mFrameCaptureRequested = true; diff --git a/dom/media/MediaManager.cpp b/dom/media/MediaManager.cpp index 04722ef2292a..9cb61168fd90 100644 --- a/dom/media/MediaManager.cpp +++ b/dom/media/MediaManager.cpp @@ -4207,8 +4207,6 @@ RefPtr SourceListener::InitializeAsync() { return InitPromise::CreateAndResolve(true, __func__); } - mStream->SetPullEnabled(true); - for (DeviceState* state : {mAudioDeviceState.get(), mVideoDeviceState.get()}) { if (!state) { @@ -4221,6 +4219,15 @@ RefPtr SourceListener::InitializeAsync() { state->mDeviceEnabled = true; state->mTrackEnabled = true; state->mTrackEnabledTime = TimeStamp::Now(); + + if (state->mDevice->GetMediaSource() != + MediaSourceEnum::AudioCapture) { + // For AudioCapture mStream is a dummy stream, so we don't try to + // enable pulling - there won't be a track to enable it for. + mStream->SetPullingEnabled( + state == mAudioDeviceState.get() ? kAudioTrack : kVideoTrack, + true); + } } return InitPromise::CreateAndResolve(true, __func__); }, @@ -4295,11 +4302,12 @@ void SourceListener::Remove() { // We disable pulling before removing so we don't risk having live tracks // without a listener attached - that wouldn't produce data and would be // illegal to the graph. - mStream->SetPullEnabled(false); if (mAudioDeviceState) { + mStream->SetPullingEnabled(kAudioTrack, false); mStream->RemoveTrackListener(mAudioDeviceState->mListener, kAudioTrack); } if (mVideoDeviceState) { + mStream->SetPullingEnabled(kVideoTrack, false); mStream->RemoveTrackListener(mVideoDeviceState->mListener, kVideoTrack); } } diff --git a/dom/media/MediaStreamGraph.cpp b/dom/media/MediaStreamGraph.cpp index 98e3d9c97761..e544650a0b83 100644 --- a/dom/media/MediaStreamGraph.cpp +++ b/dom/media/MediaStreamGraph.cpp @@ -1221,12 +1221,21 @@ void MediaStreamGraphImpl::UpdateGraph(GraphTime aEndBlockingDecisions) { } else { stream->mStartBlocking = WillUnderrun(stream, aEndBlockingDecisions); - SourceMediaStream* s = stream->AsSourceStream(); - if (s && s->mPullEnabled) { +#ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED + if (SourceMediaStream* s = stream->AsSourceStream()) { for (StreamTracks::TrackIter i(s->mTracks); !i.IsEnded(); i.Next()) { if (i->IsEnded()) { continue; } + SourceMediaStream::TrackData* data; + { + MutexAutoLock lock(s->mMutex); + data = s->FindDataForTrack(i->GetID()); + } + MOZ_ASSERT(data); + if (!data->mPullingEnabled) { + continue; + } if (i->GetEnd() < stream->GraphTimeToStreamTime(aEndBlockingDecisions)) { LOG(LogLevel::Error, @@ -1246,6 +1255,7 @@ void MediaStreamGraphImpl::UpdateGraph(GraphTime aEndBlockingDecisions) { } } } +#endif /* MOZ_DIAGNOSTIC_ASSERT_ENABLED */ } } @@ -2459,7 +2469,6 @@ void MediaStream::AddMainThreadListener( SourceMediaStream::SourceMediaStream() : MediaStream(), mMutex("mozilla::media::SourceMediaStream"), - mPullEnabled(false), mFinishPending(false) {} nsresult SourceMediaStream::OpenAudioInput(CubebUtils::AudioDeviceID aID, @@ -2496,25 +2505,39 @@ void SourceMediaStream::DestroyImpl() { MediaStream::DestroyImpl(); } -void SourceMediaStream::SetPullEnabled(bool aEnabled) { +void SourceMediaStream::SetPullingEnabled(TrackID aTrackID, bool aEnabled) { class Message : public ControlMessage { public: - Message(SourceMediaStream* aStream, bool aEnabled) - : ControlMessage(nullptr), mStream(aStream), mEnabled(aEnabled) {} + Message(SourceMediaStream* aStream, TrackID aTrackID, bool aEnabled) + : ControlMessage(nullptr), + mStream(aStream), + mTrackID(aTrackID), + mEnabled(aEnabled) {} void Run() override { MutexAutoLock lock(mStream->mMutex); - mStream->mPullEnabled = mEnabled; + TrackData* data = mStream->FindDataForTrack(mTrackID); + if (!data) { + // We can't enable pulling for a track that was never added. We ignore + // this if we're disabling pulling, since shutdown sequences are + // complex. If there's truly an issue we'll have issues enabling anyway. + MOZ_ASSERT_IF(mEnabled, + mStream->mTracks.FindTrack(mTrackID) && + mStream->mTracks.FindTrack(mTrackID)->IsEnded()); + return; + } + data->mPullingEnabled = mEnabled; } SourceMediaStream* mStream; + TrackID mTrackID; bool mEnabled; }; - GraphImpl()->AppendMessage(MakeUnique(this, aEnabled)); + GraphImpl()->AppendMessage(MakeUnique(this, aTrackID, aEnabled)); } bool SourceMediaStream::PullNewData(GraphTime aDesiredUpToTime) { TRACE_AUDIO_CALLBACK_COMMENT("SourceMediaStream %p", this); MutexAutoLock lock(mMutex); - if (!mPullEnabled || mFinished) { + if (mFinished) { return false; } // Compute how much stream time we'll need assuming we don't block @@ -2526,6 +2549,9 @@ bool SourceMediaStream::PullNewData(GraphTime aDesiredUpToTime) { this, GraphImpl()->MediaTimeToSeconds(t), GraphImpl()->MediaTimeToSeconds(current))); for (const TrackData& track : mUpdateTracks) { + if (!track.mPullingEnabled) { + continue; + } if (track.mCommands & TrackEventCommand::TRACK_EVENT_ENDED) { continue; } @@ -2623,6 +2649,7 @@ void SourceMediaStream::AddTrackInternal(TrackID aID, TrackRate aRate, data->mEndOfFlushedData = 0; data->mCommands = TRACK_CREATE; data->mData = aSegment; + data->mPullingEnabled = false; ResampleAudioToGraphSampleRate(data, aSegment); if (!(aFlags & ADDTRACK_QUEUED) && GraphImpl()) { GraphImpl()->EnsureNextIteration(); diff --git a/dom/media/MediaStreamGraph.h b/dom/media/MediaStreamGraph.h index f63d49554003..cde7df5d9306 100644 --- a/dom/media/MediaStreamGraph.h +++ b/dom/media/MediaStreamGraph.h @@ -658,13 +658,14 @@ class SourceMediaStream : public MediaStream { // Main thread only /** - * Enable or disable pulling. When pulling is enabled, NotifyPull - * gets called on MediaStream/TrackListeners for this stream during the - * MediaStreamGraph control loop. Pulling is initially disabled. - * Due to unavoidable race conditions, after a call to SetPullEnabled(false) + * Enable or disable pulling for a specific track. + * When pulling is enabled, NotifyPull gets called on the corresponding + * MediaStreamTrackListeners for this stream during the MediaStreamGraph + * control loop. Pulling is initially disabled for all tracks. Due to + * unavoidable race conditions, after a call to SetPullingEnabled(false) * it is still possible for a NotifyPull to occur. */ - void SetPullEnabled(bool aEnabled); + void SetPullingEnabled(TrackID aTrackID, bool aEnabled); // Users of audio inputs go through the stream so it can track when the // last stream referencing an input goes away, so it can close the cubeb @@ -809,6 +810,8 @@ class SourceMediaStream : public MediaStream { // Each time the track updates are flushed to the media graph thread, // this is cleared. uint32_t mCommands; + // True if the producer of this track is having data pulled by the graph. + bool mPullingEnabled; }; bool NeedsMixing(); @@ -867,7 +870,6 @@ class SourceMediaStream : public MediaStream { nsTArray mUpdateTracks; nsTArray mPendingTracks; nsTArray> mDirectTrackListeners; - bool mPullEnabled; bool mFinishPending; }; diff --git a/media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp b/media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp index 4adcfc334867..1a1f679d7421 100644 --- a/media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp +++ b/media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp @@ -1680,7 +1680,7 @@ public: { if (!mListening) { mListening = true; - mSource->SetPullEnabled(true); + mSource->SetPullingEnabled(mTrackId, true); mMaybeTrackNeedsUnmute = true; } } @@ -1689,7 +1689,7 @@ public: { if (mListening) { mListening = false; - mSource->SetPullEnabled(false); + mSource->SetPullingEnabled(mTrackId, false); } }