Bug 1777264 Part 1: Make MediaDecoder aware of changed TrackInfo as it changes. r=alwu

When TrackInfo changes in the MediaFormatReader, this ensures that MediaDecoder
has the same information.

Differential Revision: https://phabricator.services.mozilla.com/D153815

dom/media/MediaDecoder.cpp

@@ -499,6 +499,23 @@ void MediaDecoder::OnNextFrameStatus(
   }
 }
 
+void MediaDecoder::OnTrackInfoUpdated(const VideoInfo& aVideoInfo,
+                                      const AudioInfo& aAudioInfo) {
+  MOZ_ASSERT(NS_IsMainThread());
+  MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
+
+  // Note that we don't check HasVideo() or HasAudio() here, because
+  // those are checks for existing validity. If we always set the values
+  // to what we receive, then we can go from not-video to video, for
+  // example.
+  mInfo->mVideo = aVideoInfo;
+  mInfo->mAudio = aAudioInfo;
+
+  Invalidate();
+
+  EnsureTelemetryReported();
+}
+
 void MediaDecoder::OnSecondaryVideoContainerInstalled(
     const RefPtr<VideoFrameContainer>& aSecondaryVideoContainer) {
   MOZ_ASSERT(NS_IsMainThread());
@@ -582,6 +599,8 @@ void MediaDecoder::SetStateMachineParameters() {
       mAbstractMainThread, this, &MediaDecoder::OnMediaNotSeekable);
   mOnNextFrameStatus = mDecoderStateMachine->OnNextFrameStatus().Connect(
       mAbstractMainThread, this, &MediaDecoder::OnNextFrameStatus);
+  mOnTrackInfoUpdated = mDecoderStateMachine->OnTrackInfoUpdatedEvent().Connect(
+      mAbstractMainThread, this, &MediaDecoder::OnTrackInfoUpdated);
   mOnSecondaryVideoContainerInstalled =
       mDecoderStateMachine->OnSecondaryVideoContainerInstalled().Connect(
           mAbstractMainThread, this,
@@ -610,6 +629,7 @@ void MediaDecoder::DisconnectEvents() {
   mOnWaitingForKey.Disconnect();
   mOnDecodeWarning.Disconnect();
   mOnNextFrameStatus.Disconnect();
+  mOnTrackInfoUpdated.Disconnect();
   mOnSecondaryVideoContainerInstalled.Disconnect();
   mOnStoreDecoderBenchmark.Disconnect();
 }

dom/media/MediaDecoder.h

@@ -498,6 +498,9 @@ class MediaDecoder : public DecoderDoctorLifeLogger<MediaDecoder> {
 
   void OnNextFrameStatus(MediaDecoderOwner::NextFrameStatus);
 
+  void OnTrackInfoUpdated(const VideoInfo& aVideoInfo,
+                          const AudioInfo& aAudioInfo);
+
   void OnSecondaryVideoContainerInstalled(
       const RefPtr<VideoFrameContainer>& aSecondaryVideoContainer);
 
@@ -601,6 +604,7 @@ class MediaDecoder : public DecoderDoctorLifeLogger<MediaDecoder> {
   MediaEventListener mOnWaitingForKey;
   MediaEventListener mOnDecodeWarning;
   MediaEventListener mOnNextFrameStatus;
+  MediaEventListener mOnTrackInfoUpdated;
   MediaEventListener mOnSecondaryVideoContainerInstalled;
   MediaEventListener mOnStoreDecoderBenchmark;
 

dom/media/MediaDecoderStateMachineBase.h

@@ -132,6 +132,10 @@ class MediaDecoderStateMachineBase {
     return mOnNextFrameStatus;
   }
 
+  MediaEventProducer<VideoInfo, AudioInfo>& OnTrackInfoUpdatedEvent() {
+    return mReader->OnTrackInfoUpdatedEvent();
+  }
+
   MediaEventSource<void>& OnMediaNotSeekable() const;
 
   AbstractCanonical<media::NullableTimeUnit>* CanonicalDuration() {

dom/media/MediaFormatReader.cpp

@@ -863,6 +863,9 @@ MediaFormatReader::MediaFormatReader(MediaFormatReaderInit& aInit,
              StaticPrefs::media_audio_max_decode_error()),
       mVideo(this, MediaData::Type::VIDEO_DATA,
              StaticPrefs::media_video_max_decode_error()),
+      mWorkingInfoChanged(false, "MediaFormatReader::mWorkingInfoChanged"),
+      mWatchManager(this, OwnerThread()),
+      mIsWatchingWorkingInfo(false),
       mDemuxer(new DemuxerProxy(aDemuxer)),
       mDemuxerInitDone(false),
       mPendingNotifyDataArrived(false),
@@ -910,6 +913,12 @@ RefPtr<ShutdownPromise> MediaFormatReader::Shutdown() {
                   "MediaFormatReader is shutting down"),
       __func__);
 
+  if (mIsWatchingWorkingInfo) {
+    mWatchManager.Unwatch(mWorkingInfoChanged,
+                          &MediaFormatReader::NotifyTrackInfoUpdated);
+  }
+  mWatchManager.Shutdown();
+
   if (mAudio.HasPromise()) {
     mAudio.RejectPromise(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
   }
@@ -977,6 +986,30 @@ void MediaFormatReader::NotifyDecoderBenchmarkStore() {
   }
 }
 
+void MediaFormatReader::NotifyTrackInfoUpdated() {
+  MOZ_ASSERT(OnTaskQueue());
+  if (mWorkingInfoChanged) {
+    mWorkingInfoChanged = false;
+
+    VideoInfo videoInfo;
+    AudioInfo audioInfo;
+    {
+      MutexAutoLock lock(mVideo.mMutex);
+      if (HasVideo()) {
+        videoInfo = *mVideo.GetWorkingInfo()->GetAsVideoInfo();
+      }
+    }
+    {
+      MutexAutoLock lock(mAudio.mMutex);
+      if (HasAudio()) {
+        audioInfo = *mAudio.GetWorkingInfo()->GetAsAudioInfo();
+      }
+    }
+
+    mTrackInfoUpdatedEvent.Notify(videoInfo, audioInfo);
+  }
+}
+
 RefPtr<ShutdownPromise> MediaFormatReader::TearDownDecoders() {
   if (mAudio.mTaskQueue) {
     mAudio.mTaskQueue->BeginShutdown();
@@ -1175,6 +1208,7 @@ void MediaFormatReader::OnDemuxerInitDone(const MediaResult& aResult) {
       for (const MetadataTag& tag : videoInfo->mTags) {
         tags->InsertOrUpdate(tag.mKey, tag.mValue);
       }
+      mWorkingInfoChanged = true;
       mVideo.mOriginalInfo = std::move(videoInfo);
       mTrackDemuxersMayBlock |= mVideo.mTrackDemuxer->GetSamplesMayBlock();
     } else {
@@ -1205,6 +1239,7 @@ void MediaFormatReader::OnDemuxerInitDone(const MediaResult& aResult) {
       for (const MetadataTag& tag : audioInfo->mTags) {
         tags->InsertOrUpdate(tag.mKey, tag.mValue);
       }
+      mWorkingInfoChanged = true;
       mAudio.mOriginalInfo = std::move(audioInfo);
       mTrackDemuxersMayBlock |= mAudio.mTrackDemuxer->GetSamplesMayBlock();
     } else {
@@ -1292,6 +1327,10 @@ void MediaFormatReader::MaybeResolveMetadataPromise() {
   mHasStartTime = true;
   UpdateBuffered();
 
+  mWatchManager.Watch(mWorkingInfoChanged,
+                      &MediaFormatReader::NotifyTrackInfoUpdated);
+  mIsWatchingWorkingInfo = true;
+
   mMetadataPromise.Resolve(std::move(metadata), __func__);
 }
 
@@ -2013,6 +2052,7 @@ void MediaFormatReader::HandleDemuxedSamples(
       } else if (aTrack == TrackInfo::kVideoTrack) {
         decoder.mWorkingInfo = MakeUnique<VideoInfo>(*info->GetAsVideoInfo());
       }
+      mWorkingInfoChanged = true;
     }
 
     decoder.mMeanRate.Reset();
@@ -2527,6 +2567,7 @@ void MediaFormatReader::ReturnOutput(MediaData* aData, TrackType aTrack) {
       MutexAutoLock lock(mAudio.mMutex);
       mAudio.mWorkingInfo->GetAsAudioInfo()->mRate = audioData->mRate;
       mAudio.mWorkingInfo->GetAsAudioInfo()->mChannels = audioData->mChannels;
+      mWorkingInfoChanged = true;
     }
     mAudio.ResolvePromise(audioData, __func__);
   } else if (aTrack == TrackInfo::kVideoTrack) {
@@ -2539,6 +2580,7 @@ void MediaFormatReader::ReturnOutput(MediaData* aData, TrackType aTrack) {
       mInfo.mVideo.mDisplay = videoData->mDisplay;
       MutexAutoLock lock(mVideo.mMutex);
       mVideo.mWorkingInfo->GetAsVideoInfo()->mDisplay = videoData->mDisplay;
+      mWorkingInfoChanged = true;
     }
 
     TimeUnit nextKeyframe;

dom/media/MediaFormatReader.h

@@ -246,6 +246,10 @@ class MediaFormatReader final
     return mOnStoreDecoderBenchmark;
   }
 
+  MediaEventProducer<VideoInfo, AudioInfo>& OnTrackInfoUpdatedEvent() {
+    return mTrackInfoUpdatedEvent;
+  }
+
  private:
   bool HasVideo() const { return mVideo.mTrackDemuxer; }
   bool HasAudio() const { return mAudio.mTrackDemuxer; }
@@ -335,6 +339,8 @@ class MediaFormatReader final
   // This is called only on TaskQueue.
   void NotifyDecoderBenchmarkStore();
 
+  void NotifyTrackInfoUpdated();
+
   enum class DrainState {
     None,
     DrainRequested,
@@ -683,6 +689,10 @@ class MediaFormatReader final
   DecoderDataWithPromise<AudioData> mAudio;
   DecoderDataWithPromise<VideoData> mVideo;
 
+  Watchable<bool> mWorkingInfoChanged;
+  WatchManager<MediaFormatReader> mWatchManager;
+  bool mIsWatchingWorkingInfo;
+
   // Returns true when the decoder for this track needs input.
   bool NeedInput(DecoderData& aDecoder);
 
@@ -822,6 +832,8 @@ class MediaFormatReader final
 
   MediaEventProducer<VideoInfo> mOnStoreDecoderBenchmark;
 
+  MediaEventProducer<VideoInfo, AudioInfo> mTrackInfoUpdatedEvent;
+
   RefPtr<FrameStatistics> mFrameStats;
 
   // Used in bug 1393399 for telemetry.

dom/media/ReaderProxy.h

@@ -64,6 +64,9 @@ class ReaderProxy {
   MediaEventSource<void>& OnMediaNotSeekable() {
     return mReader->OnMediaNotSeekable();
   }
+  MediaEventProducer<VideoInfo, AudioInfo>& OnTrackInfoUpdatedEvent() {
+    return mReader->OnTrackInfoUpdatedEvent();
+  }
   size_t SizeOfAudioQueueInFrames() const {
     return mReader->SizeOfAudioQueueInFrames();
   }