Bug 1743834 - Create and manage the lifetime of the AudioStream EndedPromise in AudioSinkWrapper. r=alwu,media-playback-reviewers

We're going to shutdown the AudioStream in a subsequent patch, when audio is
muted. This will allow resolving it at the right time when the media ends while
muted.

This also extracts a method to start an AudioSink, because it's going to be
started in multiple locations in a future patch.

Differential Revision: https://phabricator.services.mozilla.com/D136233

dom/media/AudioStream.cpp

@@ -320,19 +320,18 @@ void AudioStream::SetStreamName(const nsAString& aStreamName) {
   }
 }
 
-Result<already_AddRefed<MediaSink::EndedPromise>, nsresult>
-AudioStream::Start() {
+nsresult AudioStream::Start(
+    MozPromiseHolder<MediaSink::EndedPromise>& aEndedPromise) {
   TRACE("AudioStream::Start");
   MOZ_ASSERT(mState == INITIALIZED);
   mState = STARTED;
-
   RefPtr<MediaSink::EndedPromise> promise;
   {
     MonitorAutoLock mon(mMonitor);
     // As cubeb might call audio stream's state callback very soon after we
     // start cubeb, we have to create the promise beforehand in order to handle
     // the case where we immediately get `drained`.
-    promise = mEndedPromise.Ensure(__func__);
+    mEndedPromise = std::move(aEndedPromise);
     mPlaybackComplete = false;
 
     if (InvokeCubeb(cubeb_stream_start) != CUBEB_OK) {
@@ -344,9 +343,9 @@ AudioStream::Start() {
                            : mState == DRAINED ? "DRAINED"
                                                : "ERRORED");
   if (mState == STARTED || mState == DRAINED) {
-    return promise.forget();
+    return NS_OK;
   }
-  return Err(NS_ERROR_FAILURE);
+  return NS_ERROR_FAILURE;
 }
 
 void AudioStream::Pause() {
@@ -391,7 +390,8 @@ void AudioStream::Resume() {
   }
 }
 
-void AudioStream::Shutdown() {
+Maybe<MozPromiseHolder<MediaSink::EndedPromise>> AudioStream::Shutdown(
+    ShutdownCause aCause) {
   TRACE("AudioStream::Shutdown");
   LOG("Shutdown, state %d", mState.load());
 
@@ -414,7 +414,15 @@ void AudioStream::Shutdown() {
   }
 
   mState = SHUTDOWN;
-  mEndedPromise.ResolveIfExists(true, __func__);
+  // When shutting down, if this AudioStream is shutting down because the
+  // HTMLMediaElement is now muted, hand back the ended promise, so that it can
+  // properly be resolved if the end of the media is reached while muted (i.e.
+  // without having an AudioStream)
+  if (aCause != ShutdownCause::Muting) {
+    mEndedPromise.ResolveIfExists(true, __func__);
+    return Nothing();
+  }
+  return Some(std::move(mEndedPromise));
 }
 
 int64_t AudioStream::GetPosition() {

dom/media/AudioStream.h

@@ -35,6 +35,13 @@ struct CubebDestroyPolicy {
   }
 };
 
+enum class ShutdownCause {
+  // Regular shutdown, signal the end of the audio stream.
+  Regular,
+  // Shutdown for muting, don't signal the end of the audio stream.
+  Muting
+};
+
 class AudioStream;
 class FrameHistory;
 class AudioConfig;
@@ -245,7 +252,8 @@ class AudioStream final {
   nsresult Init(AudioDeviceInfo* aSinkInfo);
 
   // Closes the stream. All future use of the stream is an error.
-  void Shutdown();
+  Maybe<MozPromiseHolder<MediaSink::EndedPromise>> Shutdown(
+      ShutdownCause = ShutdownCause::Regular);
 
   void Reset();
 
@@ -255,9 +263,8 @@ class AudioStream final {
 
   void SetStreamName(const nsAString& aStreamName);
 
-  // Start the stream and return a promise that will be resolve when the
-  // playback completes.
-  Result<already_AddRefed<MediaSink::EndedPromise>, nsresult> Start();
+  // Start the stream.
+  nsresult Start(MozPromiseHolder<MediaSink::EndedPromise>& aEndedPromise);
 
   // Pause audio playback.
   void Pause();

dom/media/mediasink/AudioSink.cpp

@@ -66,8 +66,9 @@ AudioSink::AudioSink(AbstractThread* aThread,
 
 AudioSink::~AudioSink() = default;
 
-Result<already_AddRefed<MediaSink::EndedPromise>, nsresult> AudioSink::Start(
-    const PlaybackParams& aParams) {
+nsresult AudioSink::Start(
+    const PlaybackParams& aParams,
+    MozPromiseHolder<MediaSink::EndedPromise>& aEndedPromise) {
   MOZ_ASSERT(mOwnerThread->IsCurrentThreadIn());
 
   mAudioQueueListener = mAudioQueue.PushEvent().Connect(
@@ -85,7 +86,7 @@ Result<already_AddRefed<MediaSink::EndedPromise>, nsresult> AudioSink::Start(
     return Err(rv);
   }
 
-  return mAudioStream->Start();
+  return mAudioStream->Start(aEndedPromise);
 }
 
 TimeUnit AudioSink::GetPosition() {
@@ -119,18 +120,60 @@ TimeUnit AudioSink::UnplayedDuration() const {
   return TimeUnit::FromMicroseconds(AudioQueuedInRingBufferMS());
 }
 
-void AudioSink::Shutdown() {
+void AudioSink::ReenqueueUnplayedAudioDataIfNeeded() {
+  // This is OK: the AudioStream has been shut down. Shutdown guarantees that
+  // the audio callback thread won't call back again.
+  mProcessedSPSCQueue->ResetThreadIds();
+
+  // construct an AudioData
+  int sampleCount = mProcessedSPSCQueue->AvailableRead();
+  uint32_t channelCount = mConverter->OutputConfig().Channels();
+  uint32_t rate = mConverter->OutputConfig().Rate();
+  uint32_t frameCount = sampleCount / channelCount;
+
+  auto duration = FramesToTimeUnit(frameCount, rate);
+  if (!duration.IsValid()) {
+    NS_WARNING("Int overflow in AudioSink");
+    mErrored = true;
+    return;
+  }
+
+  AlignedAudioBuffer queuedAudio(sampleCount);
+  DebugOnly<int> samplesRead =
+    mProcessedSPSCQueue->Dequeue(queuedAudio.Data(), sampleCount);
+  MOZ_ASSERT(samplesRead == sampleCount);
+
+  // Extrapolate mOffset, mTime from the front of the queue
+  // We can't really find a good value for `mOffset`, so we take what we have
+  // at the front of the queue.
+  // For `mTime`, assume there hasn't been a discontinuity recently.
+  RefPtr<AudioData> frontPacket = mAudioQueue.PeekFront();
+  RefPtr<AudioData> data =
+      new AudioData(frontPacket->mOffset, frontPacket->mTime - duration, std::move(queuedAudio),
+                    channelCount, rate);
+  MOZ_DIAGNOSTIC_ASSERT(duration == data->mDuration, "must be equal");
+
+  mAudioQueue.PushFront(data);
+}
+
+Maybe<MozPromiseHolder<MediaSink::EndedPromise>> AudioSink::Shutdown(
+    ShutdownCause aShutdownCause) {
   MOZ_ASSERT(mOwnerThread->IsCurrentThreadIn());
 
   mAudioQueueListener.Disconnect();
   mAudioQueueFinishListener.Disconnect();
   mProcessedQueueListener.Disconnect();
 
+  Maybe<MozPromiseHolder<MediaSink::EndedPromise>> rv;
+
   if (mAudioStream) {
-    mAudioStream->Shutdown();
+    rv = mAudioStream->Shutdown(aShutdownCause);
     mAudioStream = nullptr;
+    ReenqueueUnplayedAudioDataIfNeeded();
   }
   mProcessedQueueFinished = true;
+
+  return rv;
 }
 
 void AudioSink::SetVolume(double aVolume) {

dom/media/mediasink/AudioSink.h

@@ -42,10 +42,9 @@ class AudioSink : private AudioStream::DataSource {
 
   ~AudioSink();
 
-  // Start audio playback and return a promise which will be resolved when the
-  // playback finishes, or return an error result if any error occurs.
-  Result<already_AddRefed<MediaSink::EndedPromise>, nsresult> Start(
-      const PlaybackParams& aParams);
+  // Start audio playback.
+  nsresult Start(const PlaybackParams& aParams,
+                 MozPromiseHolder<MediaSink::EndedPromise>& aEndedPromise);
 
   /*
    * All public functions are not thread-safe.
@@ -61,8 +60,10 @@ class AudioSink : private AudioStream::DataSource {
   // The duration of the buffered frames.
   media::TimeUnit UnplayedDuration() const;
 
-  // Shut down the AudioSink's resources.
-  void Shutdown();
+  // Shut down the AudioSink's resources. If an AudioStream existed, return the
+  // ended promise it had, if it's shutting down-mid stream becaues it's muting.
+  Maybe<MozPromiseHolder<MediaSink::EndedPromise>> Shutdown(
+      ShutdownCause aShutdownCause = ShutdownCause::Regular);
 
   void SetVolume(double aVolume);
   void SetStreamName(const nsAString& aStreamName);
@@ -87,6 +88,15 @@ class AudioSink : private AudioStream::DataSource {
                      bool aAudioThreadChanged) override;
   bool Ended() const override;
 
+  // When shutting down, it's important to not lose any audio data, it might be
+  // still of use, in two scenarios:
+  // - If the audio is now captured to a MediaStream, whatever is enqueued in
+  // the ring buffer needs to be played out now ;
+  // - If the AudioSink is shutting down because the audio is muted, it's
+  // important to keep the audio around in case it's quickly unmuted,
+  // and in general to keep A/V sync correct when unmuted.
+  void ReenqueueUnplayedAudioDataIfNeeded();
+
   void CheckIsAudible(const Span<AudioDataValue>& aInterleaved,
                       size_t aChannel);
 

dom/media/mediasink/AudioSinkWrapper.cpp

@@ -27,6 +27,7 @@ void AudioSinkWrapper::Shutdown() {
   AssertOwnerThread();
   MOZ_ASSERT(!mIsStarted, "Must be called after playback stopped.");
   mCreator = nullptr;
+  mEndedPromiseHolder.ResolveIfExists(true, __func__);
 }
 
 RefPtr<MediaSink::EndedPromise> AudioSinkWrapper::OnEnded(TrackType aType) {
@@ -188,21 +189,32 @@ nsresult AudioSinkWrapper::Start(const TimeUnit& aStartTime,
     return NS_OK;
   }
 
+  nsresult rv = StartAudioSink(aStartTime);
+
+  return rv;
+}
+
+nsresult AudioSinkWrapper::StartAudioSink(const TimeUnit& aStartTime) {
+  MOZ_RELEASE_ASSERT(!mAudioSink);
+
+  RefPtr<MediaSink::EndedPromise> promise =
+      mEndedPromiseHolder.Ensure(__func__);
+
   mAudioSink.reset(mCreator->Create(aStartTime));
-  Result<already_AddRefed<MediaSink::EndedPromise>, nsresult> rv =
-      mAudioSink->Start(mParams);
-  if (rv.isErr()) {
-    mEndedPromise =
-        MediaSink::EndedPromise::CreateAndReject(rv.unwrapErr(), __func__);
+  nsresult rv = mAudioSink->Start(mParams, mEndedPromiseHolder);
+  if (NS_FAILED(rv)) {
+    mEndedPromise = MediaSink::EndedPromise::CreateAndReject(rv, __func__);
   } else {
-    mEndedPromise = rv.unwrap();
+    mEndedPromise = promise;
   }
 
+  mAudioSinkEndedPromise.DisconnectIfExists();
   mEndedPromise
       ->Then(mOwnerThread.get(), __func__, this,
              &AudioSinkWrapper::OnAudioEnded, &AudioSinkWrapper::OnAudioEnded)
       ->Track(mAudioSinkEndedPromise);
-  return rv.isErr() ? rv.unwrapErr() : NS_OK;
+
+  return rv;
 }
 
 bool AudioSinkWrapper::IsAudioSourceEnded(const MediaInfo& aInfo) const {
@@ -221,7 +233,9 @@ void AudioSinkWrapper::Stop() {
 
   if (mAudioSink) {
     mAudioSinkEndedPromise.DisconnectIfExists();
-    mAudioSink->Shutdown();
+    DebugOnly<Maybe<MozPromiseHolder<EndedPromise>>> rv =
+        mAudioSink->Shutdown();
+    MOZ_ASSERT(rv.inspect().isNothing());
     mAudioSink = nullptr;
     mEndedPromise = nullptr;
   }

dom/media/mediasink/AudioSinkWrapper.h

@@ -92,6 +92,8 @@ class AudioSinkWrapper : public MediaSink {
     MOZ_ASSERT(mOwnerThread->IsCurrentThreadIn());
   }
 
+  nsresult StartAudioSink(const media::TimeUnit& aStartTime);
+
   media::TimeUnit GetVideoPosition(TimeStamp aNow) const;
 
   void OnAudioEnded();
@@ -103,6 +105,7 @@ class AudioSinkWrapper : public MediaSink {
   UniquePtr<AudioSink> mAudioSink;
   // Will only exist when media has an audio track.
   RefPtr<EndedPromise> mEndedPromise;
+  MozPromiseHolder<EndedPromise> mEndedPromiseHolder;
 
   bool mIsStarted;
   PlaybackParams mParams;