diff --git a/dom/media/AudioStream.cpp b/dom/media/AudioStream.cpp index ea32bf289882..8aa2cc0e5bfd 100644 --- a/dom/media/AudioStream.cpp +++ b/dom/media/AudioStream.cpp @@ -320,19 +320,18 @@ void AudioStream::SetStreamName(const nsAString& aStreamName) { } } -Result, nsresult> -AudioStream::Start() { +nsresult AudioStream::Start( + MozPromiseHolder& aEndedPromise) { TRACE("AudioStream::Start"); MOZ_ASSERT(mState == INITIALIZED); mState = STARTED; - RefPtr promise; { MonitorAutoLock mon(mMonitor); // As cubeb might call audio stream's state callback very soon after we // start cubeb, we have to create the promise beforehand in order to handle // the case where we immediately get `drained`. - promise = mEndedPromise.Ensure(__func__); + mEndedPromise = std::move(aEndedPromise); mPlaybackComplete = false; if (InvokeCubeb(cubeb_stream_start) != CUBEB_OK) { @@ -344,9 +343,9 @@ AudioStream::Start() { : mState == DRAINED ? "DRAINED" : "ERRORED"); if (mState == STARTED || mState == DRAINED) { - return promise.forget(); + return NS_OK; } - return Err(NS_ERROR_FAILURE); + return NS_ERROR_FAILURE; } void AudioStream::Pause() { @@ -391,7 +390,8 @@ void AudioStream::Resume() { } } -void AudioStream::Shutdown() { +Maybe> AudioStream::Shutdown( + ShutdownCause aCause) { TRACE("AudioStream::Shutdown"); LOG("Shutdown, state %d", mState.load()); @@ -414,7 +414,15 @@ void AudioStream::Shutdown() { } mState = SHUTDOWN; - mEndedPromise.ResolveIfExists(true, __func__); + // When shutting down, if this AudioStream is shutting down because the + // HTMLMediaElement is now muted, hand back the ended promise, so that it can + // properly be resolved if the end of the media is reached while muted (i.e. + // without having an AudioStream) + if (aCause != ShutdownCause::Muting) { + mEndedPromise.ResolveIfExists(true, __func__); + return Nothing(); + } + return Some(std::move(mEndedPromise)); } int64_t AudioStream::GetPosition() { diff --git a/dom/media/AudioStream.h b/dom/media/AudioStream.h index f6c7d0b4ef36..04ed62493dcc 100644 --- a/dom/media/AudioStream.h +++ b/dom/media/AudioStream.h @@ -35,6 +35,13 @@ struct CubebDestroyPolicy { } }; +enum class ShutdownCause { + // Regular shutdown, signal the end of the audio stream. + Regular, + // Shutdown for muting, don't signal the end of the audio stream. + Muting +}; + class AudioStream; class FrameHistory; class AudioConfig; @@ -245,7 +252,8 @@ class AudioStream final { nsresult Init(AudioDeviceInfo* aSinkInfo); // Closes the stream. All future use of the stream is an error. - void Shutdown(); + Maybe> Shutdown( + ShutdownCause = ShutdownCause::Regular); void Reset(); @@ -255,9 +263,8 @@ class AudioStream final { void SetStreamName(const nsAString& aStreamName); - // Start the stream and return a promise that will be resolve when the - // playback completes. - Result, nsresult> Start(); + // Start the stream. + nsresult Start(MozPromiseHolder& aEndedPromise); // Pause audio playback. void Pause(); diff --git a/dom/media/mediasink/AudioSink.cpp b/dom/media/mediasink/AudioSink.cpp index d6bdd6667673..b011c4c9ad46 100644 --- a/dom/media/mediasink/AudioSink.cpp +++ b/dom/media/mediasink/AudioSink.cpp @@ -66,8 +66,9 @@ AudioSink::AudioSink(AbstractThread* aThread, AudioSink::~AudioSink() = default; -Result, nsresult> AudioSink::Start( - const PlaybackParams& aParams) { +nsresult AudioSink::Start( + const PlaybackParams& aParams, + MozPromiseHolder& aEndedPromise) { MOZ_ASSERT(mOwnerThread->IsCurrentThreadIn()); mAudioQueueListener = mAudioQueue.PushEvent().Connect( @@ -85,7 +86,7 @@ Result, nsresult> AudioSink::Start( return Err(rv); } - return mAudioStream->Start(); + return mAudioStream->Start(aEndedPromise); } TimeUnit AudioSink::GetPosition() { @@ -119,18 +120,60 @@ TimeUnit AudioSink::UnplayedDuration() const { return TimeUnit::FromMicroseconds(AudioQueuedInRingBufferMS()); } -void AudioSink::Shutdown() { +void AudioSink::ReenqueueUnplayedAudioDataIfNeeded() { + // This is OK: the AudioStream has been shut down. Shutdown guarantees that + // the audio callback thread won't call back again. + mProcessedSPSCQueue->ResetThreadIds(); + + // construct an AudioData + int sampleCount = mProcessedSPSCQueue->AvailableRead(); + uint32_t channelCount = mConverter->OutputConfig().Channels(); + uint32_t rate = mConverter->OutputConfig().Rate(); + uint32_t frameCount = sampleCount / channelCount; + + auto duration = FramesToTimeUnit(frameCount, rate); + if (!duration.IsValid()) { + NS_WARNING("Int overflow in AudioSink"); + mErrored = true; + return; + } + + AlignedAudioBuffer queuedAudio(sampleCount); + DebugOnly samplesRead = + mProcessedSPSCQueue->Dequeue(queuedAudio.Data(), sampleCount); + MOZ_ASSERT(samplesRead == sampleCount); + + // Extrapolate mOffset, mTime from the front of the queue + // We can't really find a good value for `mOffset`, so we take what we have + // at the front of the queue. + // For `mTime`, assume there hasn't been a discontinuity recently. + RefPtr frontPacket = mAudioQueue.PeekFront(); + RefPtr data = + new AudioData(frontPacket->mOffset, frontPacket->mTime - duration, std::move(queuedAudio), + channelCount, rate); + MOZ_DIAGNOSTIC_ASSERT(duration == data->mDuration, "must be equal"); + + mAudioQueue.PushFront(data); +} + +Maybe> AudioSink::Shutdown( + ShutdownCause aShutdownCause) { MOZ_ASSERT(mOwnerThread->IsCurrentThreadIn()); mAudioQueueListener.Disconnect(); mAudioQueueFinishListener.Disconnect(); mProcessedQueueListener.Disconnect(); + Maybe> rv; + if (mAudioStream) { - mAudioStream->Shutdown(); + rv = mAudioStream->Shutdown(aShutdownCause); mAudioStream = nullptr; + ReenqueueUnplayedAudioDataIfNeeded(); } mProcessedQueueFinished = true; + + return rv; } void AudioSink::SetVolume(double aVolume) { diff --git a/dom/media/mediasink/AudioSink.h b/dom/media/mediasink/AudioSink.h index 8404220376bb..74cc793da14d 100644 --- a/dom/media/mediasink/AudioSink.h +++ b/dom/media/mediasink/AudioSink.h @@ -42,10 +42,9 @@ class AudioSink : private AudioStream::DataSource { ~AudioSink(); - // Start audio playback and return a promise which will be resolved when the - // playback finishes, or return an error result if any error occurs. - Result, nsresult> Start( - const PlaybackParams& aParams); + // Start audio playback. + nsresult Start(const PlaybackParams& aParams, + MozPromiseHolder& aEndedPromise); /* * All public functions are not thread-safe. @@ -61,8 +60,10 @@ class AudioSink : private AudioStream::DataSource { // The duration of the buffered frames. media::TimeUnit UnplayedDuration() const; - // Shut down the AudioSink's resources. - void Shutdown(); + // Shut down the AudioSink's resources. If an AudioStream existed, return the + // ended promise it had, if it's shutting down-mid stream becaues it's muting. + Maybe> Shutdown( + ShutdownCause aShutdownCause = ShutdownCause::Regular); void SetVolume(double aVolume); void SetStreamName(const nsAString& aStreamName); @@ -87,6 +88,15 @@ class AudioSink : private AudioStream::DataSource { bool aAudioThreadChanged) override; bool Ended() const override; + // When shutting down, it's important to not lose any audio data, it might be + // still of use, in two scenarios: + // - If the audio is now captured to a MediaStream, whatever is enqueued in + // the ring buffer needs to be played out now ; + // - If the AudioSink is shutting down because the audio is muted, it's + // important to keep the audio around in case it's quickly unmuted, + // and in general to keep A/V sync correct when unmuted. + void ReenqueueUnplayedAudioDataIfNeeded(); + void CheckIsAudible(const Span& aInterleaved, size_t aChannel); diff --git a/dom/media/mediasink/AudioSinkWrapper.cpp b/dom/media/mediasink/AudioSinkWrapper.cpp index 8570544de58e..fb99567d1e46 100644 --- a/dom/media/mediasink/AudioSinkWrapper.cpp +++ b/dom/media/mediasink/AudioSinkWrapper.cpp @@ -27,6 +27,7 @@ void AudioSinkWrapper::Shutdown() { AssertOwnerThread(); MOZ_ASSERT(!mIsStarted, "Must be called after playback stopped."); mCreator = nullptr; + mEndedPromiseHolder.ResolveIfExists(true, __func__); } RefPtr AudioSinkWrapper::OnEnded(TrackType aType) { @@ -188,21 +189,32 @@ nsresult AudioSinkWrapper::Start(const TimeUnit& aStartTime, return NS_OK; } + nsresult rv = StartAudioSink(aStartTime); + + return rv; +} + +nsresult AudioSinkWrapper::StartAudioSink(const TimeUnit& aStartTime) { + MOZ_RELEASE_ASSERT(!mAudioSink); + + RefPtr promise = + mEndedPromiseHolder.Ensure(__func__); + mAudioSink.reset(mCreator->Create(aStartTime)); - Result, nsresult> rv = - mAudioSink->Start(mParams); - if (rv.isErr()) { - mEndedPromise = - MediaSink::EndedPromise::CreateAndReject(rv.unwrapErr(), __func__); + nsresult rv = mAudioSink->Start(mParams, mEndedPromiseHolder); + if (NS_FAILED(rv)) { + mEndedPromise = MediaSink::EndedPromise::CreateAndReject(rv, __func__); } else { - mEndedPromise = rv.unwrap(); + mEndedPromise = promise; } + mAudioSinkEndedPromise.DisconnectIfExists(); mEndedPromise ->Then(mOwnerThread.get(), __func__, this, &AudioSinkWrapper::OnAudioEnded, &AudioSinkWrapper::OnAudioEnded) ->Track(mAudioSinkEndedPromise); - return rv.isErr() ? rv.unwrapErr() : NS_OK; + + return rv; } bool AudioSinkWrapper::IsAudioSourceEnded(const MediaInfo& aInfo) const { @@ -221,7 +233,9 @@ void AudioSinkWrapper::Stop() { if (mAudioSink) { mAudioSinkEndedPromise.DisconnectIfExists(); - mAudioSink->Shutdown(); + DebugOnly>> rv = + mAudioSink->Shutdown(); + MOZ_ASSERT(rv.inspect().isNothing()); mAudioSink = nullptr; mEndedPromise = nullptr; } diff --git a/dom/media/mediasink/AudioSinkWrapper.h b/dom/media/mediasink/AudioSinkWrapper.h index 1b4ae3f30284..aa68a43746d7 100644 --- a/dom/media/mediasink/AudioSinkWrapper.h +++ b/dom/media/mediasink/AudioSinkWrapper.h @@ -92,6 +92,8 @@ class AudioSinkWrapper : public MediaSink { MOZ_ASSERT(mOwnerThread->IsCurrentThreadIn()); } + nsresult StartAudioSink(const media::TimeUnit& aStartTime); + media::TimeUnit GetVideoPosition(TimeStamp aNow) const; void OnAudioEnded(); @@ -103,6 +105,7 @@ class AudioSinkWrapper : public MediaSink { UniquePtr mAudioSink; // Will only exist when media has an audio track. RefPtr mEndedPromise; + MozPromiseHolder mEndedPromiseHolder; bool mIsStarted; PlaybackParams mParams;