diff --git a/dom/media/MediaDecoder.cpp b/dom/media/MediaDecoder.cpp
index e1457e5be1dc..4cd16bfc4c6a 100644
--- a/dom/media/MediaDecoder.cpp
+++ b/dom/media/MediaDecoder.cpp
@@ -248,7 +248,10 @@ void MediaDecoder::AddOutputStream(DOMMediaStream* aStream) {
   MOZ_ASSERT(mDecoderStateMachine, "Must be called after Load().");
   AbstractThread::AutoEnter context(AbstractMainThread());
   mDecoderStateMachine->EnsureOutputStreamManager(
-      aStream->GetInputStream()->Graph(), ToMaybe(mInfo.get()));
+      aStream->GetInputStream()->Graph());
+  if (mInfo) {
+    mDecoderStateMachine->EnsureOutputStreamManagerHasTracks(*mInfo);
+  }
   mDecoderStateMachine->AddOutputStream(aStream);
 }
 
@@ -656,6 +659,7 @@ void MediaDecoder::MetadataLoaded(
       aInfo->mMediaSeekableOnlyInBufferedRanges;
   mInfo = aInfo.release();
   GetOwner()->ConstructMediaTracks(mInfo);
+  mDecoderStateMachine->EnsureOutputStreamManagerHasTracks(*mInfo);
 
   // Make sure the element and the frame (if any) are told about
   // our new size.
diff --git a/dom/media/MediaDecoderStateMachine.cpp b/dom/media/MediaDecoderStateMachine.cpp
index 89b009261879..c21397573579 100644
--- a/dom/media/MediaDecoderStateMachine.cpp
+++ b/dom/media/MediaDecoderStateMachine.cpp
@@ -3338,8 +3338,8 @@ void MediaDecoderStateMachine::FinishDecodeFirstFrame() {
 RefPtr<ShutdownPromise> MediaDecoderStateMachine::BeginShutdown() {
   MOZ_ASSERT(NS_IsMainThread());
   if (mOutputStreamManager) {
-    mNextOutputStreamTrackID = mOutputStreamManager->NextTrackID();
     mOutputStreamManager->Disconnect();
+    mNextOutputStreamTrackID = mOutputStreamManager->NextTrackID();
   }
   return InvokeAsync(OwnerThread(), this, __func__,
                      &MediaDecoderStateMachine::Shutdown);
@@ -3790,28 +3790,39 @@ void MediaDecoderStateMachine::RemoveOutputStream(DOMMediaStream* aStream) {
 }
 
 void MediaDecoderStateMachine::EnsureOutputStreamManager(
-    MediaStreamGraph* aGraph, const Maybe<MediaInfo>& aLoadedInfo) {
+    MediaStreamGraph* aGraph) {
   MOZ_ASSERT(NS_IsMainThread());
   if (mOutputStreamManager) {
     return;
   }
-  LOG("Starting output track allocations at id %d", mNextOutputStreamTrackID);
   mOutputStreamManager = new OutputStreamManager(
       aGraph->CreateSourceStream(), mNextOutputStreamTrackID,
       mOutputStreamPrincipal, mOutputStreamCORSMode, mAbstractMainThread);
-  if (!aLoadedInfo) {
+}
+
+void MediaDecoderStateMachine::EnsureOutputStreamManagerHasTracks(
+    const MediaInfo& aLoadedInfo) {
+  MOZ_ASSERT(NS_IsMainThread());
+  if (!mOutputStreamManager) {
     return;
   }
-  TrackID mirroredTrackIDAllocation = mNextOutputStreamTrackID;
-  if (aLoadedInfo->HasAudio()) {
-    mOutputStreamManager->AddTrack(mirroredTrackIDAllocation++,
-                                   MediaSegment::AUDIO);
-    LOG("Pre-created audio track with id %d", mirroredTrackIDAllocation - 1);
+  if ((!aLoadedInfo.HasAudio() ||
+       mOutputStreamManager->HasTrackType(MediaSegment::AUDIO)) &&
+      (!aLoadedInfo.HasVideo() ||
+       mOutputStreamManager->HasTrackType(MediaSegment::VIDEO))) {
+    return;
   }
-  if (aLoadedInfo->HasVideo()) {
-    mOutputStreamManager->AddTrack(mirroredTrackIDAllocation++,
-                                   MediaSegment::VIDEO);
-    LOG("Pre-created video track with id %d", mirroredTrackIDAllocation - 1);
+  if (aLoadedInfo.HasAudio()) {
+    MOZ_ASSERT(!mOutputStreamManager->HasTrackType(MediaSegment::AUDIO));
+    mOutputStreamManager->AddTrack(MediaSegment::AUDIO);
+    LOG("Pre-created audio track with id %d",
+        mOutputStreamManager->GetLiveTrackIDFor(MediaSegment::AUDIO));
+  }
+  if (aLoadedInfo.HasVideo()) {
+    MOZ_ASSERT(!mOutputStreamManager->HasTrackType(MediaSegment::VIDEO));
+    mOutputStreamManager->AddTrack(MediaSegment::VIDEO);
+    LOG("Pre-created video track with id %d",
+        mOutputStreamManager->GetLiveTrackIDFor(MediaSegment::VIDEO));
   }
 }
 
diff --git a/dom/media/MediaDecoderStateMachine.h b/dom/media/MediaDecoderStateMachine.h
index a71d0368b8d5..ea874deb5624 100644
--- a/dom/media/MediaDecoderStateMachine.h
+++ b/dom/media/MediaDecoderStateMachine.h
@@ -186,11 +186,11 @@ class MediaDecoderStateMachine
 
   void SetOutputStreamPrincipal(const nsCOMPtr<nsIPrincipal>& aPrincipal);
   void SetOutputStreamCORSMode(CORSMode aCORSMode);
-  // If an OutputStreamManager does not exist, one will be created and tracks
-  // matching aLoadedInfo will be created ahead of being created by the
-  // DecodedStream sink.
-  void EnsureOutputStreamManager(MediaStreamGraph* aGraph,
-                                 const Maybe<MediaInfo>& aLoadedInfo);
+  // If an OutputStreamManager does not exist, one will be created.
+  void EnsureOutputStreamManager(MediaStreamGraph* aGraph);
+  // If an OutputStreamManager exists, tracks matching aLoadedInfo will be
+  // created unless they already exist in the manager.
+  void EnsureOutputStreamManagerHasTracks(const MediaInfo& aLoadedInfo);
   // Add an output stream to the output stream manager. The manager must have
   // been created through EnsureOutputStreamManager() before this.
   void AddOutputStream(DOMMediaStream* aStream);
diff --git a/dom/media/mediasink/DecodedStream.cpp b/dom/media/mediasink/DecodedStream.cpp
index 0448ffbfe1dc..e51eec591293 100644
--- a/dom/media/mediasink/DecodedStream.cpp
+++ b/dom/media/mediasink/DecodedStream.cpp
@@ -317,32 +317,14 @@ DecodedStreamData::DecodedStreamData(
       mOutputStreamManager(aOutputStreamManager),
       mAbstractMainThread(aMainThread) {
   MOZ_ASSERT(NS_IsMainThread());
-  // Initialize tracks on main thread and in the MediaStreamGraph.
-  // Tracks on main thread may have been created early in OutputStreamManager
-  // by the state machine, since creating them here is async from the js call.
-  // If they were pre-created in OutputStreamManager and the MediaInfo has
-  // changed since then, we end them and create new tracks.
-  if (!mOutputStreamManager->HasTracks(aInit.mAudioTrackID,
-                                       aInit.mVideoTrackID)) {
-    // Because these tracks were pre-allocated, we also have to increment the
-    // internal track allocator by the same number of tracks, so we don't risk
-    // a TrackID collision.
-    for (size_t i = 0; i < mOutputStreamManager->NumberOfTracks(); ++i) {
-      Unused << mOutputStreamManager->AllocateNextTrackID();
-    }
-    mOutputStreamManager->RemoveTracks();
-  }
+  MOZ_DIAGNOSTIC_ASSERT(
+      mOutputStreamManager->HasTracks(aInit.mAudioTrackID, aInit.mVideoTrackID),
+      "Tracks must be pre-created on main thread");
   if (IsTrackIDExplicit(aInit.mAudioTrackID)) {
-    if (!mOutputStreamManager->HasTrack(aInit.mAudioTrackID)) {
-      mOutputStreamManager->AddTrack(aInit.mAudioTrackID, MediaSegment::AUDIO);
-    }
     mStream->AddAudioTrack(aInit.mAudioTrackID, aInit.mInfo.mAudio.mRate,
                            new AudioSegment());
   }
   if (IsTrackIDExplicit(aInit.mVideoTrackID)) {
-    if (!mOutputStreamManager->HasTrack(aInit.mVideoTrackID)) {
-      mOutputStreamManager->AddTrack(aInit.mVideoTrackID, MediaSegment::VIDEO);
-    }
     mStream->AddTrack(aInit.mVideoTrackID, new VideoSegment());
   }
 }
@@ -457,12 +439,18 @@ nsresult DecodedStream::Start(const TimeUnit& aStartTime,
         mVideoEndedPromise.Resolve(true, __func__);
         return NS_OK;
       }
-      mInit.mAudioTrackID = mInit.mInfo.HasAudio()
-                                ? mOutputStreamManager->AllocateNextTrackID()
-                                : TRACK_NONE;
-      mInit.mVideoTrackID = mInit.mInfo.HasVideo()
-                                ? mOutputStreamManager->AllocateNextTrackID()
-                                : TRACK_NONE;
+      if (mInit.mInfo.HasAudio() &&
+          !mOutputStreamManager->HasTrackType(MediaSegment::AUDIO)) {
+        mOutputStreamManager->AddTrack(MediaSegment::AUDIO);
+      }
+      if (mInit.mInfo.HasVideo() &&
+          !mOutputStreamManager->HasTrackType(MediaSegment::VIDEO)) {
+        mOutputStreamManager->AddTrack(MediaSegment::VIDEO);
+      }
+      mInit.mAudioTrackID =
+          mOutputStreamManager->GetLiveTrackIDFor(MediaSegment::AUDIO);
+      mInit.mVideoTrackID =
+          mOutputStreamManager->GetLiveTrackIDFor(MediaSegment::VIDEO);
       mData = MakeUnique<DecodedStreamData>(
           mOutputStreamManager, std::move(mInit), std::move(mAudioEndedPromise),
           std::move(mVideoEndedPromise), mAbstractMainThread);
@@ -542,9 +530,12 @@ void DecodedStream::DestroyData(UniquePtr<DecodedStreamData>&& aData) {
 
   mOutputListener.Disconnect();
 
-  NS_DispatchToMainThread(
-      NS_NewRunnableFunction("DecodedStream::DestroyData",
-                             [data = std::move(aData)]() { data->Forget(); }));
+  NS_DispatchToMainThread(NS_NewRunnableFunction(
+      "DecodedStream::DestroyData",
+      [data = std::move(aData), manager = mOutputStreamManager]() {
+        data->Forget();
+        manager->RemoveTracks();
+      }));
 }
 
 void DecodedStream::SetPlaying(bool aPlaying) {
diff --git a/dom/media/mediasink/OutputStreamManager.cpp b/dom/media/mediasink/OutputStreamManager.cpp
index 5a1dad8d1d6b..3d4e443f5655 100644
--- a/dom/media/mediasink/OutputStreamManager.cpp
+++ b/dom/media/mediasink/OutputStreamManager.cpp
@@ -201,10 +201,10 @@ void OutputStreamManager::Remove(DOMMediaStream* aDOMStream) {
   MOZ_ASSERT(rv);
 }
 
-bool OutputStreamManager::HasTrack(TrackID aTrackID) {
+bool OutputStreamManager::HasTrackType(MediaSegment::Type aType) {
   MOZ_ASSERT(NS_IsMainThread());
 
-  return mLiveTracks.Contains(aTrackID, TrackIDComparator());
+  return mLiveTracks.Contains(aType, TrackTypeComparator());
 }
 
 bool OutputStreamManager::HasTracks(TrackID aAudioTrack, TrackID aVideoTrack) {
@@ -216,13 +216,13 @@ bool OutputStreamManager::HasTracks(TrackID aAudioTrack, TrackID aVideoTrack) {
     Unused << ++nrExpectedTracks;
     asExpected = asExpected && mLiveTracks.Contains(
                                    MakePair(aAudioTrack, MediaSegment::AUDIO),
-                                   TrackTypeComparator());
+                                   TrackComparator());
   }
   if (IsTrackIDExplicit(aVideoTrack)) {
     Unused << ++nrExpectedTracks;
     asExpected = asExpected && mLiveTracks.Contains(
                                    MakePair(aVideoTrack, MediaSegment::VIDEO),
-                                   TrackTypeComparator());
+                                   TrackComparator());
   }
   asExpected = asExpected && mLiveTracks.Length() == nrExpectedTracks;
   return asExpected;
@@ -233,17 +233,20 @@ size_t OutputStreamManager::NumberOfTracks() {
   return mLiveTracks.Length();
 }
 
-void OutputStreamManager::AddTrack(TrackID aTrackID, MediaSegment::Type aType) {
+void OutputStreamManager::AddTrack(MediaSegment::Type aType) {
   MOZ_ASSERT(NS_IsMainThread());
   MOZ_ASSERT(!mSourceStream->IsDestroyed());
-  MOZ_ASSERT(!HasTrack(aTrackID));
+  MOZ_ASSERT(!HasTrackType(aType),
+             "Cannot have two tracks of the same type at the same time");
+
+  TrackID id = mNextTrackID++;
 
   LOG(LogLevel::Info, "Adding %s track with id %d",
-      aType == MediaSegment::AUDIO ? "audio" : "video", aTrackID);
+      aType == MediaSegment::AUDIO ? "audio" : "video", id);
 
-  mLiveTracks.AppendElement(MakePair(aTrackID, aType));
+  mLiveTracks.AppendElement(MakePair(id, aType));
   for (const auto& data : mStreams) {
-    data->AddTrack(aTrackID, aType, mPrincipal, mCORSMode, true);
+    data->AddTrack(id, aType, mPrincipal, mCORSMode, true);
   }
 }
 
@@ -259,22 +262,16 @@ void OutputStreamManager::RemoveTrack(TrackID aTrackID) {
 }
 
 void OutputStreamManager::RemoveTracks() {
-  MOZ_ASSERT(NS_IsMainThread());
-  MOZ_ASSERT(!mSourceStream->IsDestroyed());
-  for (const Pair<TrackID, MediaSegment::Type>& pair : mLiveTracks) {
-    for (const auto& data : mStreams) {
-      data->RemoveTrack(pair.first());
-    }
-  }
-  mLiveTracks.Clear();
-}
-
-void OutputStreamManager::Disconnect() {
   MOZ_ASSERT(NS_IsMainThread());
   nsTArray<Pair<TrackID, MediaSegment::Type>> liveTracks(mLiveTracks);
   for (const auto& pair : liveTracks) {
     RemoveTrack(pair.first());
   }
+}
+
+void OutputStreamManager::Disconnect() {
+  MOZ_ASSERT(NS_IsMainThread());
+  RemoveTracks();
   MOZ_ASSERT(mLiveTracks.IsEmpty());
   nsTArray<RefPtr<DOMMediaStream>> domStreams(mStreams.Length());
   for (const auto& data : mStreams) {
@@ -311,10 +308,14 @@ TrackID OutputStreamManager::NextTrackID() const {
   return mNextTrackID;
 }
 
-TrackID OutputStreamManager::AllocateNextTrackID() {
+TrackID OutputStreamManager::GetLiveTrackIDFor(MediaSegment::Type aType) const {
   MOZ_ASSERT(NS_IsMainThread());
-  MOZ_RELEASE_ASSERT(IsTrackIDExplicit(mNextTrackID));
-  return mNextTrackID++;
+  for (const auto& pair : mLiveTracks) {
+    if (pair.second() == aType) {
+      return pair.first();
+    }
+  }
+  return TRACK_NONE;
 }
 
 void OutputStreamManager::SetPlaying(bool aPlaying) {
diff --git a/dom/media/mediasink/OutputStreamManager.h b/dom/media/mediasink/OutputStreamManager.h
index 6748724ad313..c51f006e2ad0 100644
--- a/dom/media/mediasink/OutputStreamManager.h
+++ b/dom/media/mediasink/OutputStreamManager.h
@@ -76,18 +76,16 @@ class OutputStreamManager {
   void Add(DOMMediaStream* aDOMStream);
   // Remove the output stream from the collection.
   void Remove(DOMMediaStream* aDOMStream);
-  // Returns true if aTrackID has been added to all output streams.
-  bool HasTrack(TrackID aTrackID);
+  // Returns true if there's a live track of the given type.
+  bool HasTrackType(MediaSegment::Type aType);
   // Returns true if the given tracks and no others are currently live.
   // Use a non-explicit TrackID to make it ignored for that type.
   bool HasTracks(TrackID aAudioTrack, TrackID aVideoTrack);
   // Returns the number of live tracks.
   size_t NumberOfTracks();
-  // Add aTrackID to all output streams.
-  void AddTrack(TrackID aTrackID, MediaSegment::Type aType);
-  // Remove aTrackID from all output streams.
-  void RemoveTrack(TrackID aTrackID);
-  // Remove all added tracks from all output streams.
+  // Add a track to all output streams.
+  void AddTrack(MediaSegment::Type aType);
+  // Remove all currently live tracks from all output streams.
   void RemoveTracks();
   // Disconnect mSourceStream from all output streams.
   void Disconnect();
@@ -100,11 +98,11 @@ class OutputStreamManager {
   // Called when the CORSMode for the media element owning the decoder has
   // changed.
   void SetCORSMode(CORSMode aCORSMode);
-  // Returns the track id that would be used the next time a track is allocated.
+  // Returns the track id that would be used the next time a track is added.
   TrackID NextTrackID() const;
-  // Like NextTrackID() but advances internal state, so the next call returns a
-  // new unique TrackID.
-  TrackID AllocateNextTrackID();
+  // Returns the TrackID for the currently live track of the given type, or
+  // TRACK_NONE otherwise.
+  TrackID GetLiveTrackIDFor(MediaSegment::Type aType) const;
   // Called by DecodedStream when its playing state changes. While not playing
   // we suspend mSourceStream.
   void SetPlaying(bool aPlaying);
@@ -134,12 +132,22 @@ class OutputStreamManager {
     }
   };
   struct TrackTypeComparator {
+    static bool Equals(const Pair<TrackID, MediaSegment::Type>& aLiveTrack,
+                       MediaSegment::Type aType) {
+      return aLiveTrack.second() == aType;
+    }
+  };
+  struct TrackComparator {
     static bool Equals(const Pair<TrackID, MediaSegment::Type>& aLiveTrack,
                        const Pair<TrackID, MediaSegment::Type>& aOther) {
       return aLiveTrack.first() == aOther.first() &&
              aLiveTrack.second() == aOther.second();
     }
   };
+
+  // Remove aTrackID from all output streams.
+  void RemoveTrack(TrackID aTrackID);
+
   nsTArray<UniquePtr<OutputStreamData>> mStreams;
   nsTArray<Pair<TrackID, MediaSegment::Type>> mLiveTracks;
   Canonical<PrincipalHandle> mPrincipalHandle;