diff --git a/content/media/MediaDecoder.cpp b/content/media/MediaDecoder.cpp
index e42a1b5589ab..beb402be80a9 100644
--- a/content/media/MediaDecoder.cpp
+++ b/content/media/MediaDecoder.cpp
@@ -200,29 +200,20 @@ MediaDecoder::DecodedStreamData::DecodedStreamData(MediaDecoder* aDecoder,
     mHaveBlockedForPlayState(false),
     mHaveBlockedForStateMachineNotPlaying(false)
 {
-  mStream->AddMainThreadListener(this);
-  mListener = new DecodedStreamGraphListener(mStream);
+  mListener = new DecodedStreamGraphListener(mStream, this);
   mStream->AddListener(mListener);
 }
 
 MediaDecoder::DecodedStreamData::~DecodedStreamData()
 {
-  mStream->RemoveMainThreadListener(this);
   mListener->Forget();
   mStream->Destroy();
 }
 
-void
-MediaDecoder::DecodedStreamData::NotifyMainThreadStateChanged()
-{
-  mDecoder->NotifyDecodedStreamMainThreadStateChanged();
-  if (mStream->IsFinished()) {
-    mListener->SetFinishedOnMainThread(true);
-  }
-}
-
-MediaDecoder::DecodedStreamGraphListener::DecodedStreamGraphListener(MediaStream* aStream)
-  : mMutex("MediaDecoder::DecodedStreamData::mMutex"),
+MediaDecoder::DecodedStreamGraphListener::DecodedStreamGraphListener(MediaStream* aStream,
+                                                                     DecodedStreamData* aData)
+  : mData(aData),
+    mMutex("MediaDecoder::DecodedStreamData::mMutex"),
     mStream(aStream),
     mLastOutputTime(aStream->GetCurrentTime()),
     mStreamFinishedOnMainThread(false)
@@ -239,6 +230,29 @@ MediaDecoder::DecodedStreamGraphListener::NotifyOutput(MediaStreamGraph* aGraph,
   }
 }
 
+void
+MediaDecoder::DecodedStreamGraphListener::DoNotifyFinished()
+{
+  if (mData && mData->mDecoder) {
+    if (mData->mDecoder->GetState() == PLAY_STATE_PLAYING) {
+      nsCOMPtr<nsIRunnable> event =
+        NS_NewRunnableMethod(mData->mDecoder, &MediaDecoder::PlaybackEnded);
+      NS_DispatchToCurrentThread(event);
+    }
+  }
+
+  MutexAutoLock lock(mMutex);
+  mStreamFinishedOnMainThread = true;
+}
+
+void
+MediaDecoder::DecodedStreamGraphListener::NotifyFinished(MediaStreamGraph* aGraph)
+{
+  nsCOMPtr<nsIRunnable> event =
+    NS_NewRunnableMethod(this, &DecodedStreamGraphListener::DoNotifyFinished);
+  aGraph->DispatchToMainThreadAfterStreamStateUpdate(event.forget());
+}
+
 void MediaDecoder::DestroyDecodedStream()
 {
   MOZ_ASSERT(NS_IsMainThread());
@@ -326,19 +340,6 @@ void MediaDecoder::RecreateDecodedStream(int64_t aStartTimeUSecs)
   }
 }
 
-void MediaDecoder::NotifyDecodedStreamMainThreadStateChanged()
-{
-  if (mTriggerPlaybackEndedWhenSourceStreamFinishes && mDecodedStream &&
-      mDecodedStream->mStream->IsFinished()) {
-    mTriggerPlaybackEndedWhenSourceStreamFinishes = false;
-    if (GetState() == PLAY_STATE_PLAYING) {
-      nsCOMPtr<nsIRunnable> event =
-        NS_NewRunnableMethod(this, &MediaDecoder::PlaybackEnded);
-      NS_DispatchToCurrentThread(event);
-    }
-  }
-}
-
 void MediaDecoder::AddOutputStream(ProcessedMediaStream* aStream,
                                    bool aFinishWhenEnded)
 {
@@ -421,7 +422,6 @@ MediaDecoder::MediaDecoder() :
   mCalledResourceLoaded(false),
   mIgnoreProgressData(false),
   mInfiniteStream(false),
-  mTriggerPlaybackEndedWhenSourceStreamFinishes(false),
   mOwner(nullptr),
   mFrameBufferLength(0),
   mPinnedForSeek(false),
@@ -936,12 +936,6 @@ void MediaDecoder::PlaybackEnded()
   {
     ReentrantMonitorAutoEnter mon(GetReentrantMonitor());
 
-    if (mDecodedStream && !mDecodedStream->mStream->IsFinished()) {
-      // Wait for it to finish before firing PlaybackEnded()
-      mTriggerPlaybackEndedWhenSourceStreamFinishes = true;
-      return;
-    }
-
     for (int32_t i = mOutputStreams.Length() - 1; i >= 0; --i) {
       OutputStreamData& os = mOutputStreams[i];
       if (os.mStream->IsDestroyed()) {
diff --git a/content/media/MediaDecoder.h b/content/media/MediaDecoder.h
index d41a168626ed..92817556d2c3 100644
--- a/content/media/MediaDecoder.h
+++ b/content/media/MediaDecoder.h
@@ -350,13 +350,11 @@ public:
   // replaying after the input as ended. In the latter case, the new source is
   // not connected to streams created by captureStreamUntilEnded.
 
-  struct DecodedStreamData MOZ_FINAL : public MainThreadMediaStreamListener {
+  struct DecodedStreamData {
     DecodedStreamData(MediaDecoder* aDecoder,
                       int64_t aInitialTime, SourceMediaStream* aStream);
     ~DecodedStreamData();
 
-    virtual void NotifyMainThreadStateChanged() MOZ_OVERRIDE;
-
     StreamTime GetLastOutputTime() { return mListener->GetLastOutputTime(); }
     bool IsFinished() { return mListener->IsFinishedOnMainThread(); }
 
@@ -400,8 +398,11 @@ public:
 
   class DecodedStreamGraphListener : public MediaStreamListener {
   public:
-    DecodedStreamGraphListener(MediaStream* aStream);
+    DecodedStreamGraphListener(MediaStream* aStream, DecodedStreamData* aData);
     virtual void NotifyOutput(MediaStreamGraph* aGraph, GraphTime aCurrentTime) MOZ_OVERRIDE;
+    virtual void NotifyFinished(MediaStreamGraph* aGraph) MOZ_OVERRIDE;
+
+    void DoNotifyFinished();
 
     StreamTime GetLastOutputTime()
     {
@@ -410,13 +411,18 @@ public:
     }
     void Forget()
     {
+      NS_ASSERTION(NS_IsMainThread(), "Main thread only");
+      mData = nullptr;
+
       MutexAutoLock lock(mMutex);
       mStream = nullptr;
     }
-    void SetFinishedOnMainThread(bool aFinished)
+    bool SetFinishedOnMainThread(bool aFinished)
     {
       MutexAutoLock lock(mMutex);
+      bool result = !mStreamFinishedOnMainThread;
       mStreamFinishedOnMainThread = aFinished;
+      return result;
     }
     bool IsFinishedOnMainThread()
     {
@@ -424,6 +430,9 @@ public:
       return mStreamFinishedOnMainThread;
     }
   private:
+    // Main thread only
+    DecodedStreamData* mData;
+
     Mutex mMutex;
     // Protected by mMutex
     nsRefPtr<MediaStream> mStream;
@@ -465,12 +474,6 @@ public:
    * Decoder monitor must be held.
    */
   void UpdateStreamBlockingForStateMachinePlaying();
-  /**
-   * Called when the state of mDecodedStream as visible on the main thread
-   * has changed. In particular we want to know when the stream has finished
-   * so we can call PlaybackEnded.
-   */
-  void NotifyDecodedStreamMainThreadStateChanged();
   nsTArray<OutputStreamData>& OutputStreams()
   {
     GetReentrantMonitor().AssertCurrentThreadIn();
@@ -1134,10 +1137,6 @@ protected:
   // True if the stream is infinite (e.g. a webradio).
   bool mInfiniteStream;
 
-  // True if NotifyDecodedStreamMainThreadStateChanged should retrigger
-  // PlaybackEnded when mDecodedStream->mStream finishes.
-  bool mTriggerPlaybackEndedWhenSourceStreamFinishes;
-
   // Start timer to update download progress information.
   nsresult StartProgress();
 
diff --git a/content/media/MediaDecoderStateMachine.cpp b/content/media/MediaDecoderStateMachine.cpp
index bd2c68695e97..787fb860fd14 100644
--- a/content/media/MediaDecoderStateMachine.cpp
+++ b/content/media/MediaDecoderStateMachine.cpp
@@ -2369,10 +2369,14 @@ nsresult MediaDecoderStateMachine::RunStateMachine()
       }
 
       StopAudioThread();
-      if (mDecoder->GetState() == MediaDecoder::PLAY_STATE_PLAYING) {
+      // When we're decoding to a stream, the stream's main-thread finish signal
+      // will take care of calling MediaDecoder::PlaybackEnded.
+      if (mDecoder->GetState() == MediaDecoder::PLAY_STATE_PLAYING &&
+          !mDecoder->GetDecodedStream()) {
         int64_t videoTime = HasVideo() ? mVideoFrameEndTime : 0;
         int64_t clockTime = std::max(mEndTime, std::max(videoTime, GetAudioClock()));
         UpdatePlaybackPosition(clockTime);
+
         nsCOMPtr<nsIRunnable> event =
           NS_NewRunnableMethod(mDecoder, &MediaDecoder::PlaybackEnded);
         NS_DispatchToMainThread(event, NS_DISPATCH_NORMAL);
diff --git a/content/media/TrackUnionStream.h b/content/media/TrackUnionStream.h
index 1decc2279b72..12dce796a559 100644
--- a/content/media/TrackUnionStream.h
+++ b/content/media/TrackUnionStream.h
@@ -52,6 +52,9 @@ public:
     for (uint32_t i = 0; i < mInputs.Length(); ++i) {
       MediaStream* stream = mInputs[i]->GetSource();
       if (!stream->IsFinishedOnGraphThread()) {
+        // XXX we really should check whether 'stream' has finished within time aTo,
+        // not just that it's finishing when all its queued data eventually runs
+        // out.
         allFinished = false;
       }
       if (!stream->HasCurrentData()) {