/* vim:set ts=2 sw=2 sts=2 et cindent: */ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #ifdef XP_WIN // Include Windows headers required for enabling high precision timers. #include "windows.h" #include "mmsystem.h" #endif #include "mozilla/DebugOnly.h" #include #include "MediaDecoderStateMachine.h" #include "MediaTimer.h" #include "AudioSink.h" #include "nsTArray.h" #include "MediaDecoder.h" #include "MediaDecoderReader.h" #include "mozilla/MathAlgorithms.h" #include "mozilla/mozalloc.h" #include "VideoUtils.h" #include "mozilla/dom/TimeRanges.h" #include "nsDeque.h" #include "AudioSegment.h" #include "VideoSegment.h" #include "ImageContainer.h" #include "nsComponentManagerUtils.h" #include "nsITimer.h" #include "nsContentUtils.h" #include "MediaShutdownManager.h" #include "SharedThreadPool.h" #include "MediaTaskQueue.h" #include "nsIEventTarget.h" #include "prenv.h" #include "mozilla/Preferences.h" #include "gfx2DGlue.h" #include "nsPrintfCString.h" #include "DOMMediaStream.h" #include namespace mozilla { using namespace mozilla::layers; using namespace mozilla::dom; using namespace mozilla::gfx; #define NS_DispatchToMainThread(...) CompileError_UseTailDispatchInstead // avoid redefined macro in unified build #undef DECODER_LOG #undef VERBOSE_LOG #ifdef PR_LOGGING extern PRLogModuleInfo* gMediaDecoderLog; #define DECODER_LOG(x, ...) \ PR_LOG(gMediaDecoderLog, PR_LOG_DEBUG, ("Decoder=%p " x, mDecoder.get(), ##__VA_ARGS__)) #define VERBOSE_LOG(x, ...) \ PR_BEGIN_MACRO \ if (!PR_GetEnv("MOZ_QUIET")) { \ DECODER_LOG(x, ##__VA_ARGS__); \ } \ PR_END_MACRO #define SAMPLE_LOG(x, ...) \ PR_BEGIN_MACRO \ if (PR_GetEnv("MEDIA_LOG_SAMPLES")) { \ DECODER_LOG(x, ##__VA_ARGS__); \ } \ PR_END_MACRO #else #define DECODER_LOG(x, ...) #define VERBOSE_LOG(x, ...) #define SAMPLE_LOG(x, ...) #endif // Somehow MSVC doesn't correctly delete the comma before ##__VA_ARGS__ // when __VA_ARGS__ expands to nothing. This is a workaround for it. #define DECODER_WARN_HELPER(a, b) NS_WARNING b #define DECODER_WARN(x, ...) \ DECODER_WARN_HELPER(0, (nsPrintfCString("Decoder=%p " x, mDecoder.get(), ##__VA_ARGS__).get())) // GetCurrentTime is defined in winbase.h as zero argument macro forwarding to // GetTickCount() and conflicts with MediaDecoderStateMachine::GetCurrentTime // implementation. With unified builds, putting this in headers is not enough. #ifdef GetCurrentTime #undef GetCurrentTime #endif // Certain constants get stored as member variables and then adjusted by various // scale factors on a per-decoder basis. We want to make sure to avoid using these // constants directly, so we put them in a namespace. namespace detail { // If audio queue has less than this many usecs of decoded audio, we won't risk // trying to decode the video, we'll skip decoding video up to the next // keyframe. We may increase this value for an individual decoder if we // encounter video frames which take a long time to decode. static const uint32_t LOW_AUDIO_USECS = 300000; // If more than this many usecs of decoded audio is queued, we'll hold off // decoding more audio. If we increase the low audio threshold (see // LOW_AUDIO_USECS above) we'll also increase this value to ensure it's not // less than the low audio threshold. const int64_t AMPLE_AUDIO_USECS = 1000000; } // namespace detail // When we're only playing audio and we don't have a video stream, we divide // AMPLE_AUDIO_USECS and LOW_AUDIO_USECS by the following value. This reduces // the amount of decoded audio we buffer, reducing our memory usage. We only // need to decode far ahead when we're decoding video using software decoding, // as otherwise a long video decode could cause an audio underrun. const int64_t NO_VIDEO_AMPLE_AUDIO_DIVISOR = 8; // If we have fewer than LOW_VIDEO_FRAMES decoded frames, and // we're not "prerolling video", we'll skip the video up to the next keyframe // which is at or after the current playback position. static const uint32_t LOW_VIDEO_FRAMES = 1; // Threshold in usecs that used to check if we are low on decoded video. // If the last video frame's end time |mDecodedVideoEndTime| is more than // |LOW_VIDEO_THRESHOLD_USECS*mPlaybackRate| after the current clock in // Advanceframe(), the video decode is lagging, and we skip to next keyframe. static const int32_t LOW_VIDEO_THRESHOLD_USECS = 60000; // Arbitrary "frame duration" when playing only audio. static const int AUDIO_DURATION_USECS = 40000; // If we increase our "low audio threshold" (see LOW_AUDIO_USECS above), we // use this as a factor in all our calculations. Increasing this will cause // us to be more likely to increase our low audio threshold, and to // increase it by more. static const int THRESHOLD_FACTOR = 2; namespace detail { // If we have less than this much undecoded data available, we'll consider // ourselves to be running low on undecoded data. We determine how much // undecoded data we have remaining using the reader's GetBuffered() // implementation. static const int64_t LOW_DATA_THRESHOLD_USECS = 5000000; // LOW_DATA_THRESHOLD_USECS needs to be greater than AMPLE_AUDIO_USECS, otherwise // the skip-to-keyframe logic can activate when we're running low on data. static_assert(LOW_DATA_THRESHOLD_USECS > AMPLE_AUDIO_USECS, "LOW_DATA_THRESHOLD_USECS is too small"); } // namespace detail // Amount of excess usecs of data to add in to the "should we buffer" calculation. static const uint32_t EXHAUSTED_DATA_MARGIN_USECS = 60000; // If we enter buffering within QUICK_BUFFER_THRESHOLD_USECS seconds of starting // decoding, we'll enter "quick buffering" mode, which exits a lot sooner than // normal buffering mode. This exists so that if the decode-ahead exhausts the // downloaded data while decode/playback is just starting up (for example // after a seek while the media is still playing, or when playing a media // as soon as it's load started), we won't necessarily stop for 30s and wait // for buffering. We may actually be able to playback in this case, so exit // buffering early and try to play. If it turns out we can't play, we'll fall // back to buffering normally. static const uint32_t QUICK_BUFFER_THRESHOLD_USECS = 2000000; namespace detail { // If we're quick buffering, we'll remain in buffering mode while we have less than // QUICK_BUFFERING_LOW_DATA_USECS of decoded data available. static const uint32_t QUICK_BUFFERING_LOW_DATA_USECS = 1000000; // If QUICK_BUFFERING_LOW_DATA_USECS is > AMPLE_AUDIO_USECS, we won't exit // quick buffering in a timely fashion, as the decode pauses when it // reaches AMPLE_AUDIO_USECS decoded data, and thus we'll never reach // QUICK_BUFFERING_LOW_DATA_USECS. static_assert(QUICK_BUFFERING_LOW_DATA_USECS <= AMPLE_AUDIO_USECS, "QUICK_BUFFERING_LOW_DATA_USECS is too large"); } // namespace detail // The amount of instability we tollerate in calls to // MediaDecoderStateMachine::UpdateEstimatedDuration(); changes of duration // less than this are ignored, as they're assumed to be the result of // instability in the duration estimation. static const uint64_t ESTIMATED_DURATION_FUZZ_FACTOR_USECS = USECS_PER_S / 2; static TimeDuration UsecsToDuration(int64_t aUsecs) { return TimeDuration::FromMicroseconds(aUsecs); } static int64_t DurationToUsecs(TimeDuration aDuration) { return static_cast(aDuration.ToSeconds() * USECS_PER_S); } static const uint32_t MIN_VIDEO_QUEUE_SIZE = 3; static const uint32_t MAX_VIDEO_QUEUE_SIZE = 10; static uint32_t sVideoQueueDefaultSize = MAX_VIDEO_QUEUE_SIZE; static uint32_t sVideoQueueHWAccelSize = MIN_VIDEO_QUEUE_SIZE; MediaDecoderStateMachine::MediaDecoderStateMachine(MediaDecoder* aDecoder, MediaDecoderReader* aReader, bool aRealTime) : mDecoder(aDecoder), mRealTime(aRealTime), mDispatchedStateMachine(false), mDelayedScheduler(this), mState(DECODER_STATE_DECODING_NONE), mPlayDuration(0), mStartTime(-1), mEndTime(-1), mDurationSet(false), mFragmentEndTime(-1), mReader(aReader), mCurrentFrameTime(0), mAudioStartTime(-1), mAudioEndTime(-1), mDecodedAudioEndTime(-1), mVideoFrameEndTime(-1), mDecodedVideoEndTime(-1), mVolume(1.0), mPlaybackRate(1.0), mPreservesPitch(true), mLowAudioThresholdUsecs(detail::LOW_AUDIO_USECS), mAmpleAudioThresholdUsecs(detail::AMPLE_AUDIO_USECS), mQuickBufferingLowDataThresholdUsecs(detail::QUICK_BUFFERING_LOW_DATA_USECS), mIsAudioPrerolling(false), mIsVideoPrerolling(false), mAudioCaptured(false), mPositionChangeQueued(false), mAudioCompleted(false), mGotDurationFromMetaData(false), mDispatchedEventToDecode(false), mStopAudioThread(true), mQuickBuffering(false), mMinimizePreroll(false), mDecodeThreadWaiting(false), mDropAudioUntilNextDiscontinuity(false), mDropVideoUntilNextDiscontinuity(false), mDecodeToSeekTarget(false), mCurrentTimeBeforeSeek(0), mLastFrameStatus(MediaDecoderOwner::NEXT_FRAME_UNINITIALIZED), mCorruptFrames(30), mDisabledHardwareAcceleration(false), mDecodingFrozenAtStateDecoding(false), mSentLoadedMetadataEvent(false), mSentFirstFrameLoadedEvent(false), mSentPlaybackEndedEvent(false) { MOZ_COUNT_CTOR(MediaDecoderStateMachine); NS_ASSERTION(NS_IsMainThread(), "Should be on main thread."); // Set up our task queue. RefPtr pool(GetMediaThreadPool()); MOZ_DIAGNOSTIC_ASSERT(pool); mTaskQueue = new MediaTaskQueue(pool.forget(), /* aAssertTailDispatch = */ true); static bool sPrefCacheInit = false; if (!sPrefCacheInit) { sPrefCacheInit = true; Preferences::AddUintVarCache(&sVideoQueueDefaultSize, "media.video-queue.default-size", MAX_VIDEO_QUEUE_SIZE); Preferences::AddUintVarCache(&sVideoQueueHWAccelSize, "media.video-queue.hw-accel-size", MIN_VIDEO_QUEUE_SIZE); } mBufferingWait = IsRealTime() ? 0 : 15; mLowDataThresholdUsecs = IsRealTime() ? 0 : detail::LOW_DATA_THRESHOLD_USECS; #ifdef XP_WIN // Ensure high precision timers are enabled on Windows, otherwise the state // machine isn't woken up at reliable intervals to set the next frame, // and we drop frames while painting. Note that multiple calls to this // function per-process is OK, provided each call is matched by a corresponding // timeEndPeriod() call. timeBeginPeriod(1); #endif } MediaDecoderStateMachine::~MediaDecoderStateMachine() { MOZ_ASSERT(NS_IsMainThread(), "Should be on main thread."); MOZ_COUNT_DTOR(MediaDecoderStateMachine); NS_ASSERTION(!mPendingWakeDecoder.get(), "WakeDecoder should have been revoked already"); mReader = nullptr; #ifdef XP_WIN timeEndPeriod(1); #endif } bool MediaDecoderStateMachine::HasFutureAudio() { AssertCurrentThreadInMonitor(); NS_ASSERTION(HasAudio(), "Should only call HasFutureAudio() when we have audio"); // We've got audio ready to play if: // 1. We've not completed playback of audio, and // 2. we either have more than the threshold of decoded audio available, or // we've completely decoded all audio (but not finished playing it yet // as per 1). return !mAudioCompleted && (AudioDecodedUsecs() > mLowAudioThresholdUsecs * mPlaybackRate || AudioQueue().IsFinished()); } bool MediaDecoderStateMachine::HaveNextFrameData() { AssertCurrentThreadInMonitor(); return (!HasAudio() || HasFutureAudio()) && (!HasVideo() || VideoQueue().GetSize() > 0); } int64_t MediaDecoderStateMachine::GetDecodedAudioDuration() { MOZ_ASSERT(OnTaskQueue()); AssertCurrentThreadInMonitor(); int64_t audioDecoded = AudioQueue().Duration(); if (mAudioEndTime != -1) { audioDecoded += mAudioEndTime - GetMediaTime(); } return audioDecoded; } void MediaDecoderStateMachine::SendStreamAudio(AudioData* aAudio, DecodedStreamData* aStream, AudioSegment* aOutput) { MOZ_ASSERT(OnTaskQueue()); AssertCurrentThreadInMonitor(); // This logic has to mimic AudioSink closely to make sure we write // the exact same silences CheckedInt64 audioWrittenOffset = aStream->mAudioFramesWritten + UsecsToFrames(mInfo.mAudio.mRate, aStream->mInitialTime + mStartTime); CheckedInt64 frameOffset = UsecsToFrames(mInfo.mAudio.mRate, aAudio->mTime); if (!audioWrittenOffset.isValid() || !frameOffset.isValid() || // ignore packet that we've already processed frameOffset.value() + aAudio->mFrames <= audioWrittenOffset.value()) { return; } if (audioWrittenOffset.value() < frameOffset.value()) { int64_t silentFrames = frameOffset.value() - audioWrittenOffset.value(); // Write silence to catch up VERBOSE_LOG("writing %lld frames of silence to MediaStream", silentFrames); AudioSegment silence; silence.InsertNullDataAtStart(silentFrames); aStream->mAudioFramesWritten += silentFrames; audioWrittenOffset += silentFrames; aOutput->AppendFrom(&silence); } MOZ_ASSERT(audioWrittenOffset.value() >= frameOffset.value()); int64_t offset = audioWrittenOffset.value() - frameOffset.value(); size_t framesToWrite = aAudio->mFrames - offset; aAudio->EnsureAudioBuffer(); nsRefPtr buffer = aAudio->mAudioBuffer; AudioDataValue* bufferData = static_cast(buffer->Data()); nsAutoTArray channels; for (uint32_t i = 0; i < aAudio->mChannels; ++i) { channels.AppendElement(bufferData + i*aAudio->mFrames + offset); } aOutput->AppendFrames(buffer.forget(), channels, framesToWrite); VERBOSE_LOG("writing %u frames of data to MediaStream for AudioData at %lld", static_cast(framesToWrite), aAudio->mTime); aStream->mAudioFramesWritten += framesToWrite; aOutput->ApplyVolume(mVolume); aStream->mNextAudioTime = aAudio->GetEndTime(); } static void WriteVideoToMediaStream(MediaStream* aStream, layers::Image* aImage, int64_t aEndMicroseconds, int64_t aStartMicroseconds, const IntSize& aIntrinsicSize, VideoSegment* aOutput) { nsRefPtr image = aImage; StreamTime duration = aStream->MicrosecondsToStreamTimeRoundDown(aEndMicroseconds) - aStream->MicrosecondsToStreamTimeRoundDown(aStartMicroseconds); aOutput->AppendFrame(image.forget(), duration, aIntrinsicSize); } void MediaDecoderStateMachine::SendStreamData() { MOZ_ASSERT(OnTaskQueue()); AssertCurrentThreadInMonitor(); MOZ_ASSERT(!mAudioSink, "Should've been stopped in RunStateMachine()"); DecodedStreamData* stream = mDecoder->GetDecodedStream(); bool finished = (!mInfo.HasAudio() || AudioQueue().IsFinished()) && (!mInfo.HasVideo() || VideoQueue().IsFinished()); if (mDecoder->IsSameOriginMedia()) { SourceMediaStream* mediaStream = stream->mStream; StreamTime endPosition = 0; if (!stream->mStreamInitialized) { if (mInfo.HasAudio()) { TrackID audioTrackId = mInfo.mAudio.mTrackInfo.mOutputId; AudioSegment* audio = new AudioSegment(); mediaStream->AddAudioTrack(audioTrackId, mInfo.mAudio.mRate, 0, audio, SourceMediaStream::ADDTRACK_QUEUED); stream->mStream->DispatchWhenNotEnoughBuffered(audioTrackId, TaskQueue(), GetWakeDecoderRunnable()); stream->mNextAudioTime = mStartTime + stream->mInitialTime; } if (mInfo.HasVideo()) { TrackID videoTrackId = mInfo.mVideo.mTrackInfo.mOutputId; VideoSegment* video = new VideoSegment(); mediaStream->AddTrack(videoTrackId, 0, video, SourceMediaStream::ADDTRACK_QUEUED); stream->mStream->DispatchWhenNotEnoughBuffered(videoTrackId, TaskQueue(), GetWakeDecoderRunnable()); // TODO: We can't initialize |mNextVideoTime| until |mStartTime| // is set. This is a good indication that DecodedStreamData is in // deep coupling with the state machine and we should move the class // into MediaDecoderStateMachine. stream->mNextVideoTime = mStartTime + stream->mInitialTime; } mediaStream->FinishAddTracks(); stream->mStreamInitialized = true; } if (mInfo.HasAudio()) { MOZ_ASSERT(stream->mNextAudioTime != -1, "Should've been initialized"); TrackID audioTrackId = mInfo.mAudio.mTrackInfo.mOutputId; nsAutoTArray,10> audio; // It's OK to hold references to the AudioData because AudioData // is ref-counted. AudioQueue().GetElementsAfter(stream->mNextAudioTime, &audio); AudioSegment output; for (uint32_t i = 0; i < audio.Length(); ++i) { SendStreamAudio(audio[i], stream, &output); } // |mNextAudioTime| is updated as we process each audio sample in // SendStreamAudio(). This is consistent with how |mNextVideoTime| // is updated for video samples. if (output.GetDuration() > 0) { mediaStream->AppendToTrack(audioTrackId, &output); } if (AudioQueue().IsFinished() && !stream->mHaveSentFinishAudio) { mediaStream->EndTrack(audioTrackId); stream->mHaveSentFinishAudio = true; } endPosition = std::max(endPosition, mediaStream->TicksToTimeRoundDown(mInfo.mAudio.mRate, stream->mAudioFramesWritten)); } if (mInfo.HasVideo()) { MOZ_ASSERT(stream->mNextVideoTime != -1, "Should've been initialized"); TrackID videoTrackId = mInfo.mVideo.mTrackInfo.mOutputId; nsAutoTArray,10> video; // It's OK to hold references to the VideoData because VideoData // is ref-counted. VideoQueue().GetElementsAfter(stream->mNextVideoTime, &video); VideoSegment output; for (uint32_t i = 0; i < video.Length(); ++i) { VideoData* v = video[i]; if (stream->mNextVideoTime < v->mTime) { VERBOSE_LOG("writing last video to MediaStream %p for %lldus", mediaStream, v->mTime - stream->mNextVideoTime); // Write last video frame to catch up. mLastVideoImage can be null here // which is fine, it just means there's no video. // TODO: |mLastVideoImage| should come from the last image rendered // by the state machine. This will avoid the black frame when capture // happens in the middle of playback (especially in th middle of a // video frame). E.g. if we have a video frame that is 30 sec long // and capture happens at 15 sec, we'll have to append a black frame // that is 15 sec long. WriteVideoToMediaStream(mediaStream, stream->mLastVideoImage, v->mTime, stream->mNextVideoTime, stream->mLastVideoImageDisplaySize, &output); stream->mNextVideoTime = v->mTime; } if (stream->mNextVideoTime < v->GetEndTime()) { VERBOSE_LOG("writing video frame %lldus to MediaStream %p for %lldus", v->mTime, mediaStream, v->GetEndTime() - stream->mNextVideoTime); WriteVideoToMediaStream(mediaStream, v->mImage, v->GetEndTime(), stream->mNextVideoTime, v->mDisplay, &output); stream->mNextVideoTime = v->GetEndTime(); stream->mLastVideoImage = v->mImage; stream->mLastVideoImageDisplaySize = v->mDisplay; } else { VERBOSE_LOG("skipping writing video frame %lldus (end %lldus) to MediaStream", v->mTime, v->GetEndTime()); } } if (output.GetDuration() > 0) { mediaStream->AppendToTrack(videoTrackId, &output); } if (VideoQueue().IsFinished() && !stream->mHaveSentFinishVideo) { mediaStream->EndTrack(videoTrackId); stream->mHaveSentFinishVideo = true; } endPosition = std::max(endPosition, mediaStream->MicrosecondsToStreamTimeRoundDown( stream->mNextVideoTime - stream->mInitialTime)); } if (!stream->mHaveSentFinish) { stream->mStream->AdvanceKnownTracksTime(endPosition); } if (finished && !stream->mHaveSentFinish) { stream->mHaveSentFinish = true; stream->mStream->Finish(); } } const auto clockTime = GetClock(); while (true) { const AudioData* a = AudioQueue().PeekFront(); // If we discard audio samples fed to the stream immediately, we will // keep decoding audio samples till the end and consume a lot of memory. // Therefore we only discard those behind the stream clock to throttle // the decoding speed. if (a && a->mTime <= clockTime) { OnAudioEndTimeUpdate(std::max(mAudioEndTime, a->GetEndTime())); nsRefPtr releaseMe = AudioQueue().PopFront(); continue; } break; } // To be consistent with AudioSink, |mAudioCompleted| is not set // until all samples are drained. if (finished && AudioQueue().GetSize() == 0) { mAudioCompleted = true; UpdateReadyState(); } } MediaDecoderStateMachine::WakeDecoderRunnable* MediaDecoderStateMachine::GetWakeDecoderRunnable() { AssertCurrentThreadInMonitor(); if (!mPendingWakeDecoder.get()) { mPendingWakeDecoder = new WakeDecoderRunnable(this); } return mPendingWakeDecoder.get(); } bool MediaDecoderStateMachine::HaveEnoughDecodedAudio(int64_t aAmpleAudioUSecs) { AssertCurrentThreadInMonitor(); if (AudioQueue().GetSize() == 0 || GetDecodedAudioDuration() < aAmpleAudioUSecs) { return false; } if (!mAudioCaptured) { return true; } DecodedStreamData* stream = mDecoder->GetDecodedStream(); if (stream && stream->mStreamInitialized && !stream->mHaveSentFinishAudio) { MOZ_ASSERT(mInfo.HasAudio()); TrackID audioTrackId = mInfo.mAudio.mTrackInfo.mOutputId; if (!stream->mStream->HaveEnoughBuffered(audioTrackId)) { return false; } stream->mStream->DispatchWhenNotEnoughBuffered(audioTrackId, TaskQueue(), GetWakeDecoderRunnable()); } return true; } bool MediaDecoderStateMachine::HaveEnoughDecodedVideo() { AssertCurrentThreadInMonitor(); if (static_cast(VideoQueue().GetSize()) < GetAmpleVideoFrames() * mPlaybackRate) { return false; } DecodedStreamData* stream = mDecoder->GetDecodedStream(); if (stream && stream->mStreamInitialized && !stream->mHaveSentFinishVideo) { MOZ_ASSERT(mInfo.HasVideo()); TrackID videoTrackId = mInfo.mVideo.mTrackInfo.mOutputId; if (!stream->mStream->HaveEnoughBuffered(videoTrackId)) { return false; } stream->mStream->DispatchWhenNotEnoughBuffered(videoTrackId, TaskQueue(), GetWakeDecoderRunnable()); } return true; } bool MediaDecoderStateMachine::NeedToDecodeVideo() { AssertCurrentThreadInMonitor(); return IsVideoDecoding() && ((mState == DECODER_STATE_SEEKING && mDecodeToSeekTarget) || (mState == DECODER_STATE_DECODING_FIRSTFRAME && IsVideoDecoding() && VideoQueue().GetSize() == 0) || (!mMinimizePreroll && !HaveEnoughDecodedVideo())); } bool MediaDecoderStateMachine::NeedToSkipToNextKeyframe() { AssertCurrentThreadInMonitor(); if (mState == DECODER_STATE_DECODING_FIRSTFRAME) { return false; } MOZ_ASSERT(mState == DECODER_STATE_DECODING || mState == DECODER_STATE_BUFFERING || mState == DECODER_STATE_SEEKING); // We are in seeking or buffering states, don't skip frame. if (!IsVideoDecoding() || mState == DECODER_STATE_BUFFERING || mState == DECODER_STATE_SEEKING) { return false; } // Don't skip frame for video-only decoded stream because the clock time of // the stream relies on the video frame. if (mAudioCaptured && !HasAudio()) { return false; } // We'll skip the video decode to the next keyframe if we're low on // audio, or if we're low on video, provided we're not running low on // data to decode. If we're running low on downloaded data to decode, // we won't start keyframe skipping, as we'll be pausing playback to buffer // soon anyway and we'll want to be able to display frames immediately // after buffering finishes. We ignore the low audio calculations for // readers that are async, as since their audio decode runs on a different // task queue it should never run low and skipping won't help their decode. bool isLowOnDecodedAudio = !mReader->IsAsync() && !mIsAudioPrerolling && IsAudioDecoding() && (GetDecodedAudioDuration() < mLowAudioThresholdUsecs * mPlaybackRate); bool isLowOnDecodedVideo = !mIsVideoPrerolling && ((GetClock() - mDecodedVideoEndTime) * mPlaybackRate > LOW_VIDEO_THRESHOLD_USECS); bool lowUndecoded = HasLowUndecodedData(); if ((isLowOnDecodedAudio || isLowOnDecodedVideo) && !lowUndecoded) { DECODER_LOG("Skipping video decode to the next keyframe lowAudio=%d lowVideo=%d lowUndecoded=%d async=%d", isLowOnDecodedAudio, isLowOnDecodedVideo, lowUndecoded, mReader->IsAsync()); return true; } return false; } bool MediaDecoderStateMachine::NeedToDecodeAudio() { AssertCurrentThreadInMonitor(); SAMPLE_LOG("NeedToDecodeAudio() isDec=%d decToTar=%d minPrl=%d seek=%d enufAud=%d", IsAudioDecoding(), mDecodeToSeekTarget, mMinimizePreroll, mState == DECODER_STATE_SEEKING, HaveEnoughDecodedAudio(mAmpleAudioThresholdUsecs * mPlaybackRate)); return IsAudioDecoding() && ((mState == DECODER_STATE_SEEKING && mDecodeToSeekTarget) || (mState == DECODER_STATE_DECODING_FIRSTFRAME && IsAudioDecoding() && AudioQueue().GetSize() == 0) || (!mMinimizePreroll && !HaveEnoughDecodedAudio(mAmpleAudioThresholdUsecs * mPlaybackRate) && (mState != DECODER_STATE_SEEKING || mDecodeToSeekTarget))); } bool MediaDecoderStateMachine::IsAudioSeekComplete() { AssertCurrentThreadInMonitor(); SAMPLE_LOG("IsAudioSeekComplete() curTarVal=%d mAudDis=%d aqFin=%d aqSz=%d", mCurrentSeek.Exists(), mDropAudioUntilNextDiscontinuity, AudioQueue().IsFinished(), AudioQueue().GetSize()); return !HasAudio() || (mCurrentSeek.Exists() && !mDropAudioUntilNextDiscontinuity && (AudioQueue().IsFinished() || AudioQueue().GetSize() > 0)); } bool MediaDecoderStateMachine::IsVideoSeekComplete() { AssertCurrentThreadInMonitor(); SAMPLE_LOG("IsVideoSeekComplete() curTarVal=%d mVidDis=%d vqFin=%d vqSz=%d", mCurrentSeek.Exists(), mDropVideoUntilNextDiscontinuity, VideoQueue().IsFinished(), VideoQueue().GetSize()); return !HasVideo() || (mCurrentSeek.Exists() && !mDropVideoUntilNextDiscontinuity && (VideoQueue().IsFinished() || VideoQueue().GetSize() > 0)); } void MediaDecoderStateMachine::OnAudioDecoded(AudioData* aAudioSample) { MOZ_ASSERT(OnTaskQueue()); ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); nsRefPtr audio(aAudioSample); MOZ_ASSERT(audio); mAudioDataRequest.Complete(); mDecodedAudioEndTime = audio->GetEndTime(); SAMPLE_LOG("OnAudioDecoded [%lld,%lld] disc=%d", (audio ? audio->mTime : -1), (audio ? audio->GetEndTime() : -1), (audio ? audio->mDiscontinuity : 0)); switch (mState) { case DECODER_STATE_DECODING_FIRSTFRAME: { Push(audio); MaybeFinishDecodeFirstFrame(); return; } case DECODER_STATE_BUFFERING: { // If we're buffering, this may be the sample we need to stop buffering. // Save it and schedule the state machine. Push(audio); ScheduleStateMachine(); return; } case DECODER_STATE_DECODING: { Push(audio); if (mIsAudioPrerolling && DonePrerollingAudio()) { StopPrerollingAudio(); } // Schedule the state machine to send stream data as soon as possible. if (mAudioCaptured) { ScheduleStateMachine(); } return; } case DECODER_STATE_SEEKING: { if (!mCurrentSeek.Exists()) { // We've received a sample from a previous decode. Discard it. return; } if (audio->mDiscontinuity) { mDropAudioUntilNextDiscontinuity = false; } if (!mDropAudioUntilNextDiscontinuity) { // We must be after the discontinuity; we're receiving samples // at or after the seek target. if (mCurrentSeek.mTarget.mType == SeekTarget::PrevSyncPoint && mCurrentSeek.mTarget.mTime > mCurrentTimeBeforeSeek && audio->mTime < mCurrentTimeBeforeSeek) { // We are doing a fastSeek, but we ended up *before* the previous // playback position. This is surprising UX, so switch to an accurate // seek and decode to the seek target. This is not conformant to the // spec, fastSeek should always be fast, but until we get the time to // change all Readers to seek to the keyframe after the currentTime // in this case, we'll just decode forward. Bug 1026330. mCurrentSeek.mTarget.mType = SeekTarget::Accurate; } if (mCurrentSeek.mTarget.mType == SeekTarget::PrevSyncPoint) { // Non-precise seek; we can stop the seek at the first sample. AudioQueue().Push(audio); } else { // We're doing an accurate seek. We must discard // MediaData up to the one containing exact seek target. if (NS_FAILED(DropAudioUpToSeekTarget(audio))) { DecodeError(); return; } } } CheckIfSeekComplete(); return; } default: { // Ignore other cases. return; } } } void MediaDecoderStateMachine::Push(AudioData* aSample) { MOZ_ASSERT(OnTaskQueue()); MOZ_ASSERT(aSample); // TODO: Send aSample to MSG and recalculate readystate before pushing, // otherwise AdvanceFrame may pop the sample before we have a chance // to reach playing. AudioQueue().Push(aSample); if (mState > DECODER_STATE_DECODING_FIRSTFRAME) { // The ready state can change when we've decoded data, so update the // ready state, so that DOM events can fire. UpdateReadyState(); DispatchDecodeTasksIfNeeded(); mDecoder->GetReentrantMonitor().NotifyAll(); } } void MediaDecoderStateMachine::Push(VideoData* aSample) { MOZ_ASSERT(OnTaskQueue()); MOZ_ASSERT(aSample); // TODO: Send aSample to MSG and recalculate readystate before pushing, // otherwise AdvanceFrame may pop the sample before we have a chance // to reach playing. VideoQueue().Push(aSample); if (mState > DECODER_STATE_DECODING_FIRSTFRAME) { // The ready state can change when we've decoded data, so update the // ready state, so that DOM events can fire. UpdateReadyState(); DispatchDecodeTasksIfNeeded(); mDecoder->GetReentrantMonitor().NotifyAll(); } } void MediaDecoderStateMachine::OnNotDecoded(MediaData::Type aType, MediaDecoderReader::NotDecodedReason aReason) { MOZ_ASSERT(OnTaskQueue()); ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); SAMPLE_LOG("OnNotDecoded (aType=%u, aReason=%u)", aType, aReason); bool isAudio = aType == MediaData::AUDIO_DATA; MOZ_ASSERT_IF(!isAudio, aType == MediaData::VIDEO_DATA); if (isAudio) { mAudioDataRequest.Complete(); } else { mVideoDataRequest.Complete(); } if (IsShutdown()) { // Already shutdown; return; } // If this is a decode error, delegate to the generic error path. if (aReason == MediaDecoderReader::DECODE_ERROR) { DecodeError(); return; } // If the decoder is waiting for data, we tell it to call us back when the // data arrives. if (aReason == MediaDecoderReader::WAITING_FOR_DATA) { MOZ_ASSERT(mReader->IsWaitForDataSupported(), "Readers that send WAITING_FOR_DATA need to implement WaitForData"); WaitRequestRef(aType).Begin(ProxyMediaCall(DecodeTaskQueue(), mReader.get(), __func__, &MediaDecoderReader::WaitForData, aType, TailDispatcher()) ->RefableThen(TaskQueue(), __func__, this, &MediaDecoderStateMachine::OnWaitForDataResolved, &MediaDecoderStateMachine::OnWaitForDataRejected, TailDispatcher())); return; } if (aReason == MediaDecoderReader::CANCELED) { DispatchDecodeTasksIfNeeded(); return; } // This is an EOS. Finish off the queue, and then handle things based on our // state. MOZ_ASSERT(aReason == MediaDecoderReader::END_OF_STREAM); if (!isAudio && mState == DECODER_STATE_SEEKING && mCurrentSeek.Exists() && mFirstVideoFrameAfterSeek) { // Null sample. Hit end of stream. If we have decoded a frame, // insert it into the queue so that we have something to display. // We make sure to do this before invoking VideoQueue().Finish() // below. VideoQueue().Push(mFirstVideoFrameAfterSeek); mFirstVideoFrameAfterSeek = nullptr; } if (isAudio) { AudioQueue().Finish(); StopPrerollingAudio(); } else { VideoQueue().Finish(); StopPrerollingVideo(); } switch (mState) { case DECODER_STATE_DECODING_FIRSTFRAME: { MaybeFinishDecodeFirstFrame(); return; } case DECODER_STATE_BUFFERING: case DECODER_STATE_DECODING: { CheckIfDecodeComplete(); // The ready state can change when we've decoded data, so update the // ready state, so that DOM events can fire. UpdateReadyState(); mDecoder->GetReentrantMonitor().NotifyAll(); // Schedule the state machine to notify track ended as soon as possible. if (mAudioCaptured) { ScheduleStateMachine(); } return; } case DECODER_STATE_SEEKING: { if (!mCurrentSeek.Exists()) { // We've received a sample from a previous decode. Discard it. return; } if (isAudio) { mDropAudioUntilNextDiscontinuity = false; } else { mDropVideoUntilNextDiscontinuity = false; } CheckIfSeekComplete(); return; } default: { return; } } } void MediaDecoderStateMachine::MaybeFinishDecodeFirstFrame() { MOZ_ASSERT(OnTaskQueue()); AssertCurrentThreadInMonitor(); if ((IsAudioDecoding() && AudioQueue().GetSize() == 0) || (IsVideoDecoding() && VideoQueue().GetSize() == 0)) { return; } if (NS_FAILED(FinishDecodeFirstFrame())) { DecodeError(); } } void MediaDecoderStateMachine::OnVideoDecoded(VideoData* aVideoSample) { MOZ_ASSERT(OnTaskQueue()); ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); nsRefPtr video(aVideoSample); mVideoDataRequest.Complete(); mDecodedVideoEndTime = video ? video->GetEndTime() : mDecodedVideoEndTime; SAMPLE_LOG("OnVideoDecoded [%lld,%lld] disc=%d", (video ? video->mTime : -1), (video ? video->GetEndTime() : -1), (video ? video->mDiscontinuity : 0)); switch (mState) { case DECODER_STATE_DECODING_FIRSTFRAME: { Push(video); MaybeFinishDecodeFirstFrame(); return; } case DECODER_STATE_BUFFERING: { // If we're buffering, this may be the sample we need to stop buffering. // Save it and schedule the state machine. Push(video); ScheduleStateMachine(); return; } case DECODER_STATE_DECODING: { Push(video); if (mIsVideoPrerolling && DonePrerollingVideo()) { StopPrerollingVideo(); } // For non async readers, if the requested video sample was slow to // arrive, increase the amount of audio we buffer to ensure that we // don't run out of audio. This is unnecessary for async readers, // since they decode audio and video on different threads so they // are unlikely to run out of decoded audio. if (mReader->IsAsync()) { return; } TimeDuration decodeTime = TimeStamp::Now() - mVideoDecodeStartTime; if (THRESHOLD_FACTOR * DurationToUsecs(decodeTime) > mLowAudioThresholdUsecs && !HasLowUndecodedData()) { mLowAudioThresholdUsecs = std::min(THRESHOLD_FACTOR * DurationToUsecs(decodeTime), mAmpleAudioThresholdUsecs); mAmpleAudioThresholdUsecs = std::max(THRESHOLD_FACTOR * mLowAudioThresholdUsecs, mAmpleAudioThresholdUsecs); DECODER_LOG("Slow video decode, set mLowAudioThresholdUsecs=%lld mAmpleAudioThresholdUsecs=%lld", mLowAudioThresholdUsecs, mAmpleAudioThresholdUsecs); } // Schedule the state machine to send stream data as soon as possible. if (mAudioCaptured) { ScheduleStateMachine(); } return; } case DECODER_STATE_SEEKING: { if (!mCurrentSeek.Exists()) { // We've received a sample from a previous decode. Discard it. return; } if (mDropVideoUntilNextDiscontinuity) { if (video->mDiscontinuity) { mDropVideoUntilNextDiscontinuity = false; } } if (!mDropVideoUntilNextDiscontinuity) { // We must be after the discontinuity; we're receiving samples // at or after the seek target. if (mCurrentSeek.mTarget.mType == SeekTarget::PrevSyncPoint && mCurrentSeek.mTarget.mTime > mCurrentTimeBeforeSeek && video->mTime < mCurrentTimeBeforeSeek) { // We are doing a fastSeek, but we ended up *before* the previous // playback position. This is surprising UX, so switch to an accurate // seek and decode to the seek target. This is not conformant to the // spec, fastSeek should always be fast, but until we get the time to // change all Readers to seek to the keyframe after the currentTime // in this case, we'll just decode forward. Bug 1026330. mCurrentSeek.mTarget.mType = SeekTarget::Accurate; } if (mCurrentSeek.mTarget.mType == SeekTarget::PrevSyncPoint) { // Non-precise seek; we can stop the seek at the first sample. VideoQueue().Push(video); } else { // We're doing an accurate seek. We still need to discard // MediaData up to the one containing exact seek target. if (NS_FAILED(DropVideoUpToSeekTarget(video))) { DecodeError(); return; } } } CheckIfSeekComplete(); return; } default: { // Ignore other cases. return; } } } void MediaDecoderStateMachine::CheckIfSeekComplete() { MOZ_ASSERT(OnTaskQueue()); AssertCurrentThreadInMonitor(); const bool videoSeekComplete = IsVideoSeekComplete(); if (HasVideo() && !videoSeekComplete) { // We haven't reached the target. Ensure we have requested another sample. if (NS_FAILED(EnsureVideoDecodeTaskQueued())) { DECODER_WARN("Failed to request video during seek"); DecodeError(); } } const bool audioSeekComplete = IsAudioSeekComplete(); if (HasAudio() && !audioSeekComplete) { // We haven't reached the target. Ensure we have requested another sample. if (NS_FAILED(EnsureAudioDecodeTaskQueued())) { DECODER_WARN("Failed to request audio during seek"); DecodeError(); } } SAMPLE_LOG("CheckIfSeekComplete() audioSeekComplete=%d videoSeekComplete=%d", audioSeekComplete, videoSeekComplete); if (audioSeekComplete && videoSeekComplete) { mDecodeToSeekTarget = false; nsCOMPtr task( NS_NewRunnableMethod(this, &MediaDecoderStateMachine::SeekCompleted)); TailDispatch(TaskQueue(), task.forget()); } } bool MediaDecoderStateMachine::IsAudioDecoding() { AssertCurrentThreadInMonitor(); return HasAudio() && !AudioQueue().IsFinished(); } bool MediaDecoderStateMachine::IsVideoDecoding() { AssertCurrentThreadInMonitor(); return HasVideo() && !VideoQueue().IsFinished(); } void MediaDecoderStateMachine::CheckIfDecodeComplete() { MOZ_ASSERT(OnTaskQueue()); AssertCurrentThreadInMonitor(); if (IsShutdown() || mState == DECODER_STATE_SEEKING || mState == DECODER_STATE_COMPLETED) { // Don't change our state if we've already been shutdown, or we're seeking, // since we don't want to abort the shutdown or seek processes. return; } if (!IsVideoDecoding() && !IsAudioDecoding()) { // We've finished decoding all active streams, // so move to COMPLETED state. SetState(DECODER_STATE_COMPLETED); DispatchDecodeTasksIfNeeded(); ScheduleStateMachine(); } DECODER_LOG("CheckIfDecodeComplete %scompleted", ((mState == DECODER_STATE_COMPLETED) ? "" : "NOT ")); } bool MediaDecoderStateMachine::IsPlaying() const { AssertCurrentThreadInMonitor(); return !mPlayStartTime.IsNull(); } nsresult MediaDecoderStateMachine::Init(MediaDecoderStateMachine* aCloneDonor) { MOZ_ASSERT(NS_IsMainThread()); if (NS_WARN_IF(!mReader->EnsureTaskQueue())) { return NS_ERROR_FAILURE; } MediaDecoderReader* cloneReader = nullptr; if (aCloneDonor) { cloneReader = aCloneDonor->mReader; } nsresult rv = mReader->Init(cloneReader); NS_ENSURE_SUCCESS(rv, rv); return NS_OK; } void MediaDecoderStateMachine::StopPlayback() { MOZ_ASSERT(OnTaskQueue()); DECODER_LOG("StopPlayback()"); AssertCurrentThreadInMonitor(); mDecoder->NotifyPlaybackStopped(); if (IsPlaying()) { mPlayDuration = GetClock() - mStartTime; SetPlayStartTime(TimeStamp()); } // Notify the audio sink, so that it notices that we've stopped playing, // so it can pause audio playback. mDecoder->GetReentrantMonitor().NotifyAll(); NS_ASSERTION(!IsPlaying(), "Should report not playing at end of StopPlayback()"); mDecoder->UpdateStreamBlockingForStateMachinePlaying(); DispatchDecodeTasksIfNeeded(); } void MediaDecoderStateMachine::MaybeStartPlayback() { MOZ_ASSERT(OnTaskQueue()); AssertCurrentThreadInMonitor(); if (IsPlaying()) { // Logging this case is really spammy - don't do it. return; } bool playStatePermits = mDecoder->GetState() == MediaDecoder::PLAY_STATE_PLAYING; bool decodeStatePermits = mState == DECODER_STATE_DECODING || mState == DECODER_STATE_COMPLETED; if (!playStatePermits || !decodeStatePermits || mIsAudioPrerolling || mIsVideoPrerolling) { DECODER_LOG("Not starting playback [playStatePermits: %d, decodeStatePermits: %d, " "mIsAudioPrerolling: %d, mIsVideoPrerolling: %d]", (int) playStatePermits, (int) decodeStatePermits, (int) mIsAudioPrerolling, (int) mIsVideoPrerolling); return; } if (mDecoder->CheckDecoderCanOffloadAudio()) { DECODER_LOG("Offloading playback"); return; } DECODER_LOG("MaybeStartPlayback() starting playback"); mDecoder->NotifyPlaybackStarted(); SetPlayStartTime(TimeStamp::Now()); MOZ_ASSERT(IsPlaying()); nsresult rv = StartAudioThread(); NS_ENSURE_SUCCESS_VOID(rv); mDecoder->GetReentrantMonitor().NotifyAll(); mDecoder->UpdateStreamBlockingForStateMachinePlaying(); DispatchDecodeTasksIfNeeded(); } void MediaDecoderStateMachine::UpdatePlaybackPositionInternal(int64_t aTime) { MOZ_ASSERT(OnTaskQueue()); SAMPLE_LOG("UpdatePlaybackPositionInternal(%lld) (mStartTime=%lld)", aTime, mStartTime); AssertCurrentThreadInMonitor(); NS_ASSERTION(mStartTime >= 0, "Should have positive mStartTime"); mCurrentFrameTime = aTime - mStartTime; NS_ASSERTION(mCurrentFrameTime >= 0, "CurrentTime should be positive!"); if (aTime > mEndTime) { NS_ASSERTION(mCurrentFrameTime > GetDuration(), "CurrentTime must be after duration if aTime > endTime!"); DECODER_LOG("Setting new end time to %lld", aTime); mEndTime = aTime; nsCOMPtr event = NS_NewRunnableMethod(mDecoder, &MediaDecoder::DurationChanged); TailDispatch(AbstractThread::MainThread(), event.forget()); } } void MediaDecoderStateMachine::UpdatePlaybackPosition(int64_t aTime) { MOZ_ASSERT(OnTaskQueue()); UpdatePlaybackPositionInternal(aTime); bool fragmentEnded = mFragmentEndTime >= 0 && GetMediaTime() >= mFragmentEndTime; if (!mPositionChangeQueued || fragmentEnded) { mPositionChangeQueued = true; nsCOMPtr event = NS_NewRunnableMethodWithArg( mDecoder, &MediaDecoder::PlaybackPositionChanged, MediaDecoderEventVisibility::Observable); TailDispatch(AbstractThread::MainThread(), event.forget()); } mMetadataManager.DispatchMetadataIfNeeded(mDecoder, aTime); if (fragmentEnded) { StopPlayback(); } } void MediaDecoderStateMachine::ClearPositionChangeFlag() { NS_ASSERTION(NS_IsMainThread(), "Should be on main thread."); AssertCurrentThreadInMonitor(); mPositionChangeQueued = false; } MediaDecoderOwner::NextFrameStatus MediaDecoderStateMachine::GetNextFrameStatus() { ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); if (IsBuffering()) { return MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE_BUFFERING; } else if (IsSeeking()) { return MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE_SEEKING; } else if (HaveNextFrameData()) { return MediaDecoderOwner::NEXT_FRAME_AVAILABLE; } return MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE; } static const char* const gMachineStateStr[] = { "NONE", "DECODING_METADATA", "WAIT_FOR_RESOURCES", "WAIT_FOR_CDM", "DECODING_FIRSTFRAME", "DORMANT", "DECODING", "SEEKING", "BUFFERING", "COMPLETED", "SHUTDOWN", "ERROR" }; void MediaDecoderStateMachine::SetState(State aState) { MOZ_ASSERT(OnTaskQueue()); AssertCurrentThreadInMonitor(); if (mState == aState) { return; } DECODER_LOG("Change machine state from %s to %s", gMachineStateStr[mState], gMachineStateStr[aState]); mState = aState; // Clear state-scoped state. mSentPlaybackEndedEvent = false; } void MediaDecoderStateMachine::SetVolume(double volume) { NS_ASSERTION(NS_IsMainThread(), "Should be on main thread."); ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); mVolume = volume; if (mAudioSink) { mAudioSink->SetVolume(mVolume); } } void MediaDecoderStateMachine::SetAudioCaptured() { NS_ASSERTION(NS_IsMainThread(), "Should be on main thread."); AssertCurrentThreadInMonitor(); if (!mAudioCaptured) { mAudioCaptured = true; // Schedule the state machine to send stream data as soon as possible. ScheduleStateMachine(); } } double MediaDecoderStateMachine::GetCurrentTime() const { return static_cast(mCurrentFrameTime) / static_cast(USECS_PER_S); } int64_t MediaDecoderStateMachine::GetCurrentTimeUs() const { return mCurrentFrameTime; } bool MediaDecoderStateMachine::IsRealTime() const { return mRealTime; } int64_t MediaDecoderStateMachine::GetDuration() { AssertCurrentThreadInMonitor(); if (mEndTime == -1 || mStartTime == -1) return -1; return mEndTime - mStartTime; } int64_t MediaDecoderStateMachine::GetEndTime() { if (mEndTime == -1 && mDurationSet) { return INT64_MAX; } return mEndTime; } // Runnable which dispatches an event to the main thread to seek to the new // aSeekTarget. class SeekRunnable : public nsRunnable { public: SeekRunnable(MediaDecoder* aDecoder, double aSeekTarget) : mDecoder(aDecoder), mSeekTarget(aSeekTarget) {} NS_IMETHOD Run() { mDecoder->Seek(mSeekTarget, SeekTarget::Accurate); return NS_OK; } private: nsRefPtr mDecoder; double mSeekTarget; }; void MediaDecoderStateMachine::SetDuration(int64_t aDuration) { MOZ_ASSERT(NS_IsMainThread() || OnDecodeTaskQueue()); AssertCurrentThreadInMonitor(); if (aDuration < 0) { mDurationSet = false; return; } mDurationSet = true; if (mStartTime == -1) { SetStartTime(0); } if (aDuration == INT64_MAX) { mEndTime = -1; return; } mEndTime = mStartTime + aDuration; if (mDecoder && mEndTime >= 0 && mEndTime < mCurrentFrameTime) { // The current playback position is now past the end of the element duration // the user agent must also seek to the time of the end of the media // resource. if (NS_IsMainThread()) { // Seek synchronously. mDecoder->Seek(double(mEndTime) / USECS_PER_S, SeekTarget::Accurate); } else { // Queue seek to new end position. nsCOMPtr task = new SeekRunnable(mDecoder, double(mEndTime) / USECS_PER_S); AbstractThread::MainThread()->MaybeTailDispatch(task.forget()); } } } void MediaDecoderStateMachine::UpdateEstimatedDuration(int64_t aDuration) { AssertCurrentThreadInMonitor(); int64_t duration = GetDuration(); if (aDuration != duration && mozilla::Abs(aDuration - duration) > ESTIMATED_DURATION_FUZZ_FACTOR_USECS) { SetDuration(aDuration); nsCOMPtr event = NS_NewRunnableMethod(mDecoder, &MediaDecoder::DurationChanged); AbstractThread::MainThread()->MaybeTailDispatch(event.forget()); } } void MediaDecoderStateMachine::SetMediaEndTime(int64_t aEndTime) { MOZ_ASSERT(OnDecodeTaskQueue()); AssertCurrentThreadInMonitor(); mEndTime = aEndTime; } void MediaDecoderStateMachine::SetFragmentEndTime(int64_t aEndTime) { AssertCurrentThreadInMonitor(); mFragmentEndTime = aEndTime < 0 ? aEndTime : aEndTime + mStartTime; } bool MediaDecoderStateMachine::IsDormantNeeded() { return mReader->IsDormantNeeded(); } void MediaDecoderStateMachine::SetDormant(bool aDormant) { MOZ_ASSERT(OnTaskQueue()); ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); if (IsShutdown()) { return; } if (!mReader) { return; } DECODER_LOG("SetDormant=%d", aDormant); if (aDormant) { if (mState == DECODER_STATE_SEEKING) { if (mQueuedSeek.Exists()) { // Keep latest seek target } else if (mPendingSeek.Exists()) { mQueuedSeek.Steal(mPendingSeek); } else if (mCurrentSeek.Exists()) { mQueuedSeek.Steal(mCurrentSeek); } else { mQueuedSeek.mTarget = SeekTarget(mCurrentFrameTime, SeekTarget::Accurate, MediaDecoderEventVisibility::Suppressed); // XXXbholley - Nobody is listening to this promise. Do we need to pass it // back to MediaDecoder when we come out of dormant? nsRefPtr unused = mQueuedSeek.mPromise.Ensure(__func__); } } else { mQueuedSeek.mTarget = SeekTarget(mCurrentFrameTime, SeekTarget::Accurate, MediaDecoderEventVisibility::Suppressed); // XXXbholley - Nobody is listening to this promise. Do we need to pass it // back to MediaDecoder when we come out of dormant? nsRefPtr unused = mQueuedSeek.mPromise.Ensure(__func__); } mPendingSeek.RejectIfExists(__func__, TailDispatcher()); mCurrentSeek.RejectIfExists(__func__, TailDispatcher()); SetState(DECODER_STATE_DORMANT); if (IsPlaying()) { StopPlayback(); } Reset(); // Note that we do not wait for the decode task queue to go idle before // queuing the ReleaseMediaResources task - instead, we disconnect promises, // reset state, and put a ResetDecode in the decode task queue. Any tasks // that run after ResetDecode are supposed to run with a clean slate. We rely // on that in other places (i.e. seeking), so it seems reasonable to rely on // it here as well. nsCOMPtr r = NS_NewRunnableMethod(mReader, &MediaDecoderReader::ReleaseMediaResources); TailDispatch(DecodeTaskQueue(), r.forget()); // There's now no possibility of mPendingWakeDecoder being needed again. Revoke it. mPendingWakeDecoder = nullptr; mDecoder->GetReentrantMonitor().NotifyAll(); } else if ((aDormant != true) && (mState == DECODER_STATE_DORMANT)) { mDecodingFrozenAtStateDecoding = true; ScheduleStateMachine(); mCurrentFrameTime = 0; SetState(DECODER_STATE_DECODING_NONE); mDecoder->GetReentrantMonitor().NotifyAll(); } } void MediaDecoderStateMachine::Shutdown() { MOZ_ASSERT(OnTaskQueue()); // Once we've entered the shutdown state here there's no going back. ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); // Change state before issuing shutdown request to threads so those // threads can start exiting cleanly during the Shutdown call. ScheduleStateMachine(); SetState(DECODER_STATE_SHUTDOWN); if (mAudioSink) { mAudioSink->PrepareToShutdown(); } mDecoder->GetReentrantMonitor().NotifyAll(); } void MediaDecoderStateMachine::StartDecoding() { MOZ_ASSERT(OnTaskQueue()); ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); if (mState == DECODER_STATE_DECODING) { return; } SetState(DECODER_STATE_DECODING); mDecodeStartTime = TimeStamp::Now(); CheckIfDecodeComplete(); if (mState == DECODER_STATE_COMPLETED) { return; } // Reset other state to pristine values before starting decode. mIsAudioPrerolling = !DonePrerollingAudio(); mIsVideoPrerolling = !DonePrerollingVideo(); // Ensure that we've got tasks enqueued to decode data if we need to. DispatchDecodeTasksIfNeeded(); ScheduleStateMachine(); } void MediaDecoderStateMachine::NotifyWaitingForResourcesStatusChanged() { MOZ_ASSERT(OnTaskQueue()); ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); DECODER_LOG("NotifyWaitingForResourcesStatusChanged"); if (mState == DECODER_STATE_WAIT_FOR_RESOURCES) { // Try again. SetState(DECODER_STATE_DECODING_NONE); ScheduleStateMachine(); } else if (mState == DECODER_STATE_WAIT_FOR_CDM && !mReader->IsWaitingOnCDMResource()) { SetState(DECODER_STATE_DECODING_FIRSTFRAME); EnqueueDecodeFirstFrameTask(); } } void MediaDecoderStateMachine::PlayInternal() { MOZ_ASSERT(OnTaskQueue()); ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); // Once we start playing, we don't want to minimize our prerolling, as we // assume the user is likely to want to keep playing in future. This needs to // happen before we invoke StartDecoding(). if (mMinimizePreroll) { mMinimizePreroll = false; DispatchDecodeTasksIfNeeded(); } if (mDecodingFrozenAtStateDecoding) { mDecodingFrozenAtStateDecoding = false; DispatchDecodeTasksIfNeeded(); } // Some state transitions still happen synchronously on the main thread. So // if the main thread invokes Play() and then Seek(), the seek will initiate // synchronously on the main thread, and the asynchronous PlayInternal task // will arrive when it's no longer valid. The proper thing to do is to move // all state transitions to the state machine task queue, but for now we just // make sure that none of the possible main-thread state transitions (Seek(), // SetDormant(), and Shutdown()) have not occurred. if (mState != DECODER_STATE_DECODING && mState != DECODER_STATE_BUFFERING && mState != DECODER_STATE_COMPLETED) { DECODER_LOG("Unexpected state - Bailing out of PlayInternal()"); return; } // When asked to play, switch to decoding state only if // we are currently buffering. In other cases, we'll start playing anyway // when the state machine notices the decoder's state change to PLAYING. if (mState == DECODER_STATE_BUFFERING) { StartDecoding(); } ScheduleStateMachine(); } void MediaDecoderStateMachine::NotifyDataArrived(const char* aBuffer, uint32_t aLength, int64_t aOffset) { NS_ASSERTION(NS_IsMainThread(), "Only call on main thread"); mReader->NotifyDataArrived(aBuffer, aLength, aOffset); // While playing an unseekable stream of unknown duration, mEndTime is // updated (in AdvanceFrame()) as we play. But if data is being downloaded // faster than played, mEndTime won't reflect the end of playable data // since we haven't played the frame at the end of buffered data. So update // mEndTime here as new data is downloaded to prevent such a lag. // // Make sure to only do this if we have a start time, otherwise the reader // doesn't know how to compute GetBuffered. nsRefPtr buffered = new dom::TimeRanges(); if (mDecoder->IsInfinite() && (mStartTime != -1) && NS_SUCCEEDED(mDecoder->GetBuffered(buffered))) { uint32_t length = 0; buffered->GetLength(&length); if (length) { double end = 0; buffered->End(length - 1, &end); ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); mEndTime = std::max(mEndTime, end * USECS_PER_S); } } } nsRefPtr MediaDecoderStateMachine::Seek(SeekTarget aTarget) { MOZ_ASSERT(OnTaskQueue()); ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); mDecodingFrozenAtStateDecoding = false; if (IsShutdown()) { return MediaDecoder::SeekPromise::CreateAndReject(/* aIgnored = */ true, __func__); } // We need to be able to seek both at a transport level and at a media level // to seek. if (!mDecoder->IsMediaSeekable()) { DECODER_WARN("Seek() function should not be called on a non-seekable state machine"); return MediaDecoder::SeekPromise::CreateAndReject(/* aIgnored = */ true, __func__); } NS_ASSERTION(mState > DECODER_STATE_DECODING_METADATA, "We should have got duration already"); if (mState < DECODER_STATE_DECODING) { DECODER_LOG("Seek() Not Enough Data to continue at this stage, queuing seek"); mQueuedSeek.RejectIfExists(__func__, TailDispatcher()); mQueuedSeek.mTarget = aTarget; return mQueuedSeek.mPromise.Ensure(__func__); } mQueuedSeek.RejectIfExists(__func__, TailDispatcher()); mPendingSeek.RejectIfExists(__func__, TailDispatcher()); mPendingSeek.mTarget = aTarget; DECODER_LOG("Changed state to SEEKING (to %lld)", mPendingSeek.mTarget.mTime); SetState(DECODER_STATE_SEEKING); ScheduleStateMachine(); return mPendingSeek.mPromise.Ensure(__func__); } void MediaDecoderStateMachine::StopAudioThread() { MOZ_ASSERT(OnTaskQueue()); AssertCurrentThreadInMonitor(); if (mStopAudioThread) { // Audio sink is being stopped in another thread. Wait until finished. while (mAudioSink) { mDecoder->GetReentrantMonitor().Wait(); } return; } mStopAudioThread = true; // Wake up audio sink so that it can reach the finish line. mDecoder->GetReentrantMonitor().NotifyAll(); if (mAudioSink) { DECODER_LOG("Shutdown audio thread"); mAudioSink->PrepareToShutdown(); { ReentrantMonitorAutoExit exitMon(mDecoder->GetReentrantMonitor()); mAudioSink->Shutdown(); } mAudioSink = nullptr; } // Wake up those waiting for audio sink to finish. mDecoder->GetReentrantMonitor().NotifyAll(); } nsresult MediaDecoderStateMachine::EnqueueDecodeFirstFrameTask() { AssertCurrentThreadInMonitor(); MOZ_ASSERT(mState == DECODER_STATE_DECODING_FIRSTFRAME); nsCOMPtr task( NS_NewRunnableMethod(this, &MediaDecoderStateMachine::CallDecodeFirstFrame)); TailDispatch(TaskQueue(), task.forget()); return NS_OK; } void MediaDecoderStateMachine::SetReaderIdle() { MOZ_ASSERT(OnDecodeTaskQueue()); DECODER_LOG("Invoking SetReaderIdle()"); mReader->SetIdle(); } void MediaDecoderStateMachine::DispatchDecodeTasksIfNeeded() { MOZ_ASSERT(OnTaskQueue()); AssertCurrentThreadInMonitor(); if (mState != DECODER_STATE_DECODING && mState != DECODER_STATE_DECODING_FIRSTFRAME && mState != DECODER_STATE_BUFFERING && mState != DECODER_STATE_SEEKING) { return; } if (mState == DECODER_STATE_DECODING && mDecodingFrozenAtStateDecoding) { DECODER_LOG("DispatchDecodeTasksIfNeeded return due to " "mFreezeDecodingAtStateDecoding"); return; } // NeedToDecodeAudio() can go from false to true while we hold the // monitor, but it can't go from true to false. This can happen because // NeedToDecodeAudio() takes into account the amount of decoded audio // that's been written to the AudioStream but not played yet. So if we // were calling NeedToDecodeAudio() twice and we thread-context switch // between the calls, audio can play, which can affect the return value // of NeedToDecodeAudio() giving inconsistent results. So we cache the // value returned by NeedToDecodeAudio(), and make decisions // based on the cached value. If NeedToDecodeAudio() has // returned false, and then subsequently returns true and we're not // playing, it will probably be OK since we don't need to consume data // anyway. const bool needToDecodeAudio = NeedToDecodeAudio(); const bool needToDecodeVideo = NeedToDecodeVideo(); // If we're in completed state, we should not need to decode anything else. MOZ_ASSERT(mState != DECODER_STATE_COMPLETED || (!needToDecodeAudio && !needToDecodeVideo)); bool needIdle = !mDecoder->IsLogicallyPlaying() && mState != DECODER_STATE_SEEKING && !needToDecodeAudio && !needToDecodeVideo && !IsPlaying(); SAMPLE_LOG("DispatchDecodeTasksIfNeeded needAudio=%d audioStatus=%s needVideo=%d videoStatus=%s needIdle=%d", needToDecodeAudio, AudioRequestStatus(), needToDecodeVideo, VideoRequestStatus(), needIdle); if (needToDecodeAudio) { EnsureAudioDecodeTaskQueued(); } if (needToDecodeVideo) { EnsureVideoDecodeTaskQueued(); } if (needIdle) { DECODER_LOG("Dispatching SetReaderIdle() audioQueue=%lld videoQueue=%lld", GetDecodedAudioDuration(), VideoQueue().Duration()); nsCOMPtr task = NS_NewRunnableMethod( this, &MediaDecoderStateMachine::SetReaderIdle); TailDispatch(DecodeTaskQueue(), task.forget()); } } void MediaDecoderStateMachine::InitiateSeek() { MOZ_ASSERT(OnTaskQueue()); AssertCurrentThreadInMonitor(); mCurrentSeek.RejectIfExists(__func__, TailDispatcher()); mCurrentSeek.Steal(mPendingSeek); // Bound the seek time to be inside the media range. int64_t end = GetEndTime(); NS_ASSERTION(mStartTime != -1, "Should know start time by now"); NS_ASSERTION(end != -1, "Should know end time by now"); int64_t seekTime = mCurrentSeek.mTarget.mTime + mStartTime; seekTime = std::min(seekTime, end); seekTime = std::max(mStartTime, seekTime); NS_ASSERTION(seekTime >= mStartTime && seekTime <= end, "Can only seek in range [0,duration]"); mCurrentSeek.mTarget.mTime = seekTime; if (mAudioCaptured) { // TODO: We should re-create the decoded stream after seek completed as we do // for audio thread since it is until then we know which position we seek to // as far as fast-seek is concerned. It also fix the problem where stream // clock seems to go backwards during seeking. nsCOMPtr event = NS_NewRunnableMethodWithArgs(mDecoder, &MediaDecoder::RecreateDecodedStream, seekTime - mStartTime, nullptr); TailDispatch(AbstractThread::MainThread(), event.forget()); } mDropAudioUntilNextDiscontinuity = HasAudio(); mDropVideoUntilNextDiscontinuity = HasVideo(); mDecoder->StopProgressUpdates(); mCurrentTimeBeforeSeek = GetMediaTime(); // Stop playback now to ensure that while we're outside the monitor // dispatching SeekingStarted, playback doesn't advance and mess with // mCurrentFrameTime that we've setting to seekTime here. StopPlayback(); UpdatePlaybackPositionInternal(mCurrentSeek.mTarget.mTime); // SeekingStarted will do a UpdateReadyStateForData which will // inform the element and its users that we have no frames // to display nsCOMPtr startEvent = NS_NewRunnableMethodWithArg( mDecoder, &MediaDecoder::SeekingStarted, mCurrentSeek.mTarget.mEventVisibility); TailDispatch(AbstractThread::MainThread(), startEvent.forget()); // Reset our state machine and decoding pipeline before seeking. Reset(); // Do the seek. mSeekRequest.Begin(ProxyMediaCall(DecodeTaskQueue(), mReader.get(), __func__, &MediaDecoderReader::Seek, mCurrentSeek.mTarget.mTime, GetEndTime(), TailDispatcher()) ->RefableThen(TaskQueue(), __func__, this, &MediaDecoderStateMachine::OnSeekCompleted, &MediaDecoderStateMachine::OnSeekFailed, TailDispatcher())); } nsresult MediaDecoderStateMachine::DispatchAudioDecodeTaskIfNeeded() { MOZ_ASSERT(OnTaskQueue()); ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); if (IsShutdown()) { return NS_ERROR_FAILURE; } if (NeedToDecodeAudio()) { return EnsureAudioDecodeTaskQueued(); } return NS_OK; } nsresult MediaDecoderStateMachine::EnsureAudioDecodeTaskQueued() { MOZ_ASSERT(OnTaskQueue()); AssertCurrentThreadInMonitor(); SAMPLE_LOG("EnsureAudioDecodeTaskQueued isDecoding=%d status=%s", IsAudioDecoding(), AudioRequestStatus()); if (mState != DECODER_STATE_DECODING && mState != DECODER_STATE_DECODING_FIRSTFRAME && mState != DECODER_STATE_BUFFERING && mState != DECODER_STATE_SEEKING) { return NS_OK; } if (!IsAudioDecoding() || mAudioDataRequest.Exists() || mAudioWaitRequest.Exists() || mSeekRequest.Exists()) { return NS_OK; } SAMPLE_LOG("Queueing audio task - queued=%i, decoder-queued=%o", AudioQueue().GetSize(), mReader->SizeOfAudioQueueInFrames()); mAudioDataRequest.Begin(ProxyMediaCall(DecodeTaskQueue(), mReader.get(), __func__, &MediaDecoderReader::RequestAudioData, TailDispatcher()) ->RefableThen(TaskQueue(), __func__, this, &MediaDecoderStateMachine::OnAudioDecoded, &MediaDecoderStateMachine::OnAudioNotDecoded, TailDispatcher())); return NS_OK; } nsresult MediaDecoderStateMachine::DispatchVideoDecodeTaskIfNeeded() { MOZ_ASSERT(OnTaskQueue()); ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); if (IsShutdown()) { return NS_ERROR_FAILURE; } if (NeedToDecodeVideo()) { return EnsureVideoDecodeTaskQueued(); } return NS_OK; } nsresult MediaDecoderStateMachine::EnsureVideoDecodeTaskQueued() { MOZ_ASSERT(OnTaskQueue()); AssertCurrentThreadInMonitor(); SAMPLE_LOG("EnsureVideoDecodeTaskQueued isDecoding=%d status=%s", IsVideoDecoding(), VideoRequestStatus()); if (mState != DECODER_STATE_DECODING && mState != DECODER_STATE_DECODING_FIRSTFRAME && mState != DECODER_STATE_BUFFERING && mState != DECODER_STATE_SEEKING) { return NS_OK; } if (!IsVideoDecoding() || mVideoDataRequest.Exists() || mVideoWaitRequest.Exists() || mSeekRequest.Exists()) { return NS_OK; } bool skipToNextKeyFrame = NeedToSkipToNextKeyframe(); int64_t currentTime = mState == DECODER_STATE_SEEKING ? 0 : GetMediaTime(); // Time the video decode, so that if it's slow, we can increase our low // audio threshold to reduce the chance of an audio underrun while we're // waiting for a video decode to complete. mVideoDecodeStartTime = TimeStamp::Now(); SAMPLE_LOG("Queueing video task - queued=%i, decoder-queued=%o, skip=%i, time=%lld", VideoQueue().GetSize(), mReader->SizeOfVideoQueueInFrames(), skipToNextKeyFrame, currentTime); mVideoDataRequest.Begin(ProxyMediaCall(DecodeTaskQueue(), mReader.get(), __func__, &MediaDecoderReader::RequestVideoData, skipToNextKeyFrame, currentTime, TailDispatcher()) ->RefableThen(TaskQueue(), __func__, this, &MediaDecoderStateMachine::OnVideoDecoded, &MediaDecoderStateMachine::OnVideoNotDecoded, TailDispatcher())); return NS_OK; } nsresult MediaDecoderStateMachine::StartAudioThread() { MOZ_ASSERT(OnTaskQueue()); AssertCurrentThreadInMonitor(); if (mAudioCaptured) { NS_ASSERTION(mStopAudioThread, "mStopAudioThread must always be true if audio is captured"); return NS_OK; } mStopAudioThread = false; if (HasAudio() && !mAudioSink) { // The audio end time should always be at least the audio start time. mAudioEndTime = mAudioStartTime; MOZ_ASSERT(mAudioStartTime == GetMediaTime()); mAudioCompleted = false; mAudioSink = new AudioSink(this, mAudioStartTime, mInfo.mAudio, mDecoder->GetAudioChannel()); // OnAudioSinkError() will be called before Init() returns if an error // occurs during initialization. nsresult rv = mAudioSink->Init(); NS_ENSURE_SUCCESS(rv, rv); mAudioSink->SetVolume(mVolume); mAudioSink->SetPlaybackRate(mPlaybackRate); mAudioSink->SetPreservesPitch(mPreservesPitch); } return NS_OK; } int64_t MediaDecoderStateMachine::AudioDecodedUsecs() { NS_ASSERTION(HasAudio(), "Should only call AudioDecodedUsecs() when we have audio"); // The amount of audio we have decoded is the amount of audio data we've // already decoded and pushed to the hardware, plus the amount of audio // data waiting to be pushed to the hardware. int64_t pushed = (mAudioEndTime != -1) ? (mAudioEndTime - GetMediaTime()) : 0; // Currently for real time streams, AudioQueue().Duration() produce // wrong values (Bug 1114434), so we use frame counts to calculate duration. if (IsRealTime()) { return pushed + FramesToUsecs(AudioQueue().FrameCount(), mInfo.mAudio.mRate).value(); } return pushed + AudioQueue().Duration(); } bool MediaDecoderStateMachine::HasLowDecodedData(int64_t aAudioUsecs) { AssertCurrentThreadInMonitor(); MOZ_ASSERT(mReader->UseBufferingHeuristics()); // We consider ourselves low on decoded data if we're low on audio, // provided we've not decoded to the end of the audio stream, or // if we're low on video frames, provided // we've not decoded to the end of the video stream. return ((IsAudioDecoding() && AudioDecodedUsecs() < aAudioUsecs) || (IsVideoDecoding() && static_cast(VideoQueue().GetSize()) < LOW_VIDEO_FRAMES)); } bool MediaDecoderStateMachine::OutOfDecodedAudio() { return IsAudioDecoding() && !AudioQueue().IsFinished() && AudioQueue().GetSize() == 0 && (!mAudioSink || !mAudioSink->HasUnplayedFrames()); } bool MediaDecoderStateMachine::HasLowUndecodedData() { return HasLowUndecodedData(mLowDataThresholdUsecs); } bool MediaDecoderStateMachine::HasLowUndecodedData(int64_t aUsecs) { AssertCurrentThreadInMonitor(); NS_ASSERTION(mState > DECODER_STATE_DECODING_FIRSTFRAME, "Must have loaded first frame for GetBuffered() to work"); // If we don't have a duration, GetBuffered is probably not going to produce // a useful buffered range. Return false here so that we don't get stuck in // buffering mode for live streams. if (GetDuration() < 0) { return false; } nsRefPtr buffered = new dom::TimeRanges(); nsresult rv = mReader->GetBuffered(buffered.get()); NS_ENSURE_SUCCESS(rv, false); int64_t endOfDecodedVideoData = INT64_MAX; if (HasVideo() && !VideoQueue().AtEndOfStream()) { endOfDecodedVideoData = VideoQueue().Peek() ? VideoQueue().Peek()->GetEndTime() : mVideoFrameEndTime; } int64_t endOfDecodedAudioData = INT64_MAX; if (HasAudio() && !AudioQueue().AtEndOfStream()) { // mDecodedAudioEndTime could be -1 when no audio samples are decoded. // But that is fine since we consider ourself as low in decoded data when // we don't have any decoded audio samples at all. endOfDecodedAudioData = mDecodedAudioEndTime; } int64_t endOfDecodedData = std::min(endOfDecodedVideoData, endOfDecodedAudioData); return endOfDecodedData != INT64_MAX && !buffered->Contains(static_cast(endOfDecodedData) / USECS_PER_S, static_cast(std::min(endOfDecodedData + aUsecs, GetDuration())) / USECS_PER_S); } void MediaDecoderStateMachine::DecodeError() { MOZ_ASSERT(OnTaskQueue()); ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); if (IsShutdown()) { // Already shutdown. return; } // Change state to error, which will cause the state machine to wait until // the MediaDecoder shuts it down. SetState(DECODER_STATE_ERROR); ScheduleStateMachine(); DECODER_WARN("Decode error, changed state to ERROR"); // XXXbholley - Is anybody actually waiting on this monitor, or is it just // a leftover from when we used to do sync dispatch for the below? mDecoder->GetReentrantMonitor().NotifyAll(); // MediaDecoder::DecodeError notifies the owner, and then shuts down the state // machine. nsCOMPtr event = NS_NewRunnableMethod(mDecoder, &MediaDecoder::DecodeError); TailDispatch(AbstractThread::MainThread(), event.forget()); } void MediaDecoderStateMachine::OnMetadataRead(MetadataHolder* aMetadata) { ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); MOZ_ASSERT(OnTaskQueue()); MOZ_ASSERT(mState == DECODER_STATE_DECODING_METADATA); mMetadataRequest.Complete(); mDecoder->SetMediaSeekable(mReader->IsMediaSeekable()); mInfo = aMetadata->mInfo; mMetadataTags = aMetadata->mTags.forget(); if (HasVideo()) { DECODER_LOG("Video decode isAsync=%d HWAccel=%d videoQueueSize=%d", mReader->IsAsync(), mReader->VideoIsHardwareAccelerated(), GetAmpleVideoFrames()); } mDecoder->StartProgressUpdates(); mGotDurationFromMetaData = (GetDuration() != -1) || mDurationSet; if (mGotDurationFromMetaData) { // We have all the information required: duration and size // Inform the element that we've loaded the metadata. EnqueueLoadedMetadataEvent(); } if (mReader->IsWaitingOnCDMResource()) { // Metadata parsing was successful but we're still waiting for CDM caps // to become available so that we can build the correct decryptor/decoder. SetState(DECODER_STATE_WAIT_FOR_CDM); return; } SetState(DECODER_STATE_DECODING_FIRSTFRAME); EnqueueDecodeFirstFrameTask(); ScheduleStateMachine(); } void MediaDecoderStateMachine::OnMetadataNotRead(ReadMetadataFailureReason aReason) { ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); MOZ_ASSERT(OnTaskQueue()); MOZ_ASSERT(mState == DECODER_STATE_DECODING_METADATA); mMetadataRequest.Complete(); if (aReason == ReadMetadataFailureReason::WAITING_FOR_RESOURCES) { SetState(DECODER_STATE_WAIT_FOR_RESOURCES); } else { MOZ_ASSERT(aReason == ReadMetadataFailureReason::METADATA_ERROR); DECODER_WARN("Decode metadata failed, shutting down decoder"); DecodeError(); } } void MediaDecoderStateMachine::EnqueueLoadedMetadataEvent() { MOZ_ASSERT(OnTaskQueue()); nsAutoPtr info(new MediaInfo()); *info = mInfo; MediaDecoderEventVisibility visibility = mSentLoadedMetadataEvent? MediaDecoderEventVisibility::Suppressed : MediaDecoderEventVisibility::Observable; nsCOMPtr metadataLoadedEvent = new MetadataEventRunner(mDecoder, info, mMetadataTags, visibility); TailDispatch(AbstractThread::MainThread(), metadataLoadedEvent.forget()); mSentLoadedMetadataEvent = true; } void MediaDecoderStateMachine::EnqueueFirstFrameLoadedEvent() { MOZ_ASSERT(OnTaskQueue()); nsAutoPtr info(new MediaInfo()); *info = mInfo; MediaDecoderEventVisibility visibility = mSentFirstFrameLoadedEvent? MediaDecoderEventVisibility::Suppressed : MediaDecoderEventVisibility::Observable; nsCOMPtr event = new FirstFrameLoadedEventRunner(mDecoder, info, visibility); TailDispatch(AbstractThread::MainThread(), event.forget()); mSentFirstFrameLoadedEvent = true; } void MediaDecoderStateMachine::CallDecodeFirstFrame() { MOZ_ASSERT(OnTaskQueue()); ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); if (mState != DECODER_STATE_DECODING_FIRSTFRAME) { return; } if (NS_FAILED(DecodeFirstFrame())) { DECODER_WARN("Decode failed to start, shutting down decoder"); DecodeError(); } } nsresult MediaDecoderStateMachine::DecodeFirstFrame() { MOZ_ASSERT(OnTaskQueue()); AssertCurrentThreadInMonitor(); MOZ_ASSERT(mState == DECODER_STATE_DECODING_FIRSTFRAME); DECODER_LOG("DecodeFirstFrame started"); if (HasAudio()) { RefPtr decodeTask( NS_NewRunnableMethod(this, &MediaDecoderStateMachine::DispatchAudioDecodeTaskIfNeeded)); AudioQueue().AddPopListener(decodeTask, TaskQueue()); } if (HasVideo()) { RefPtr decodeTask( NS_NewRunnableMethod(this, &MediaDecoderStateMachine::DispatchVideoDecodeTaskIfNeeded)); VideoQueue().AddPopListener(decodeTask, TaskQueue()); } if (IsRealTime()) { SetStartTime(0); nsresult res = FinishDecodeFirstFrame(); NS_ENSURE_SUCCESS(res, res); } else if (mSentFirstFrameLoadedEvent) { // We're resuming from dormant state, so we don't need to request // the first samples in order to determine the media start time, // we have the start time from last time we loaded. SetStartTime(mStartTime); nsresult res = FinishDecodeFirstFrame(); NS_ENSURE_SUCCESS(res, res); } else { if (HasAudio()) { mAudioDataRequest.Begin(ProxyMediaCall(DecodeTaskQueue(), mReader.get(), __func__, &MediaDecoderReader::RequestAudioData, TailDispatcher()) ->RefableThen(TaskQueue(), __func__, this, &MediaDecoderStateMachine::OnAudioDecoded, &MediaDecoderStateMachine::OnAudioNotDecoded, TailDispatcher())); } if (HasVideo()) { mVideoDecodeStartTime = TimeStamp::Now(); mVideoDataRequest.Begin(ProxyMediaCall(DecodeTaskQueue(), mReader.get(), __func__, &MediaDecoderReader::RequestVideoData, false, int64_t(0), TailDispatcher()) ->RefableThen(TaskQueue(), __func__, this, &MediaDecoderStateMachine::OnVideoDecoded, &MediaDecoderStateMachine::OnVideoNotDecoded, TailDispatcher())); } } return NS_OK; } nsresult MediaDecoderStateMachine::FinishDecodeFirstFrame() { AssertCurrentThreadInMonitor(); MOZ_ASSERT(OnTaskQueue()); DECODER_LOG("FinishDecodeFirstFrame"); if (IsShutdown()) { return NS_ERROR_FAILURE; } if (!IsRealTime() && !mSentFirstFrameLoadedEvent) { const VideoData* v = VideoQueue().PeekFront(); const AudioData* a = AudioQueue().PeekFront(); SetStartTime(mReader->ComputeStartTime(v, a)); if (VideoQueue().GetSize()) { ReentrantMonitorAutoExit exitMon(mDecoder->GetReentrantMonitor()); RenderVideoFrame(VideoQueue().PeekFront(), TimeStamp::Now()); } } NS_ASSERTION(mStartTime != -1, "Must have start time"); MOZ_ASSERT(!(mDecoder->IsMediaSeekable() && mDecoder->IsTransportSeekable()) || (GetDuration() != -1) || mDurationSet, "Seekable media should have duration"); DECODER_LOG("Media goes from %lld to %lld (duration %lld) " "transportSeekable=%d, mediaSeekable=%d", mStartTime, mEndTime, GetDuration(), mDecoder->IsTransportSeekable(), mDecoder->IsMediaSeekable()); if (HasAudio() && !HasVideo()) { // We're playing audio only. We don't need to worry about slow video // decodes causing audio underruns, so don't buffer so much audio in // order to reduce memory usage. mAmpleAudioThresholdUsecs /= NO_VIDEO_AMPLE_AUDIO_DIVISOR; mLowAudioThresholdUsecs /= NO_VIDEO_AMPLE_AUDIO_DIVISOR; mQuickBufferingLowDataThresholdUsecs /= NO_VIDEO_AMPLE_AUDIO_DIVISOR; } // Get potentially updated metadata { ReentrantMonitorAutoExit exitMon(mDecoder->GetReentrantMonitor()); mReader->ReadUpdatedMetadata(&mInfo); } nsAutoPtr info(new MediaInfo()); *info = mInfo; if (!mGotDurationFromMetaData) { // We now have a duration, we can fire the LoadedMetadata and // FirstFrame event. EnqueueLoadedMetadataEvent(); EnqueueFirstFrameLoadedEvent(); } else { // Inform the element that we've loaded the first frame. EnqueueFirstFrameLoadedEvent(); } if (mState == DECODER_STATE_DECODING_FIRSTFRAME) { StartDecoding(); } // For very short media the first frame decode can decode the entire media. // So we need to check if this has occurred, else our decode pipeline won't // run (since it doesn't need to) and we won't detect end of stream. CheckIfDecodeComplete(); MaybeStartPlayback(); if (mQueuedSeek.Exists()) { mPendingSeek.Steal(mQueuedSeek); SetState(DECODER_STATE_SEEKING); ScheduleStateMachine(); } return NS_OK; } void MediaDecoderStateMachine::OnSeekCompleted(int64_t aTime) { ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); MOZ_ASSERT(OnTaskQueue()); mSeekRequest.Complete(); // We must decode the first samples of active streams, so we can determine // the new stream time. So dispatch tasks to do that. mDecodeToSeekTarget = true; DispatchDecodeTasksIfNeeded(); } void MediaDecoderStateMachine::OnSeekFailed(nsresult aResult) { ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); MOZ_ASSERT(OnTaskQueue()); mSeekRequest.Complete(); MOZ_ASSERT(NS_FAILED(aResult), "Cancels should also disconnect mSeekRequest"); DecodeError(); } void MediaDecoderStateMachine::SeekCompleted() { MOZ_ASSERT(OnTaskQueue()); ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); if (mState != DECODER_STATE_SEEKING) { MOZ_DIAGNOSTIC_ASSERT(mState == DECODER_STATE_DORMANT || IsShutdown()); // It would be nice to assert mCurrent.Exists() here, but it's possible that // we've transitioned to DECODER_STATE_SHUTDOWN but not yet gone through // RunStateMachine in that state, which is where this promise gets rejected. return; } int64_t seekTime = mCurrentSeek.mTarget.mTime; int64_t newCurrentTime = seekTime; // Setup timestamp state. nsRefPtr video = VideoQueue().PeekFront(); if (seekTime == mEndTime) { newCurrentTime = mAudioStartTime = seekTime; } else if (HasAudio()) { AudioData* audio = AudioQueue().PeekFront(); // Though we adjust the newCurrentTime in audio-based, and supplemented // by video. For better UX, should NOT bind the slide position to // the first audio data timestamp directly. // While seeking to a position where there's only either audio or video, or // seeking to a position lies before audio or video, we need to check if // seekTime is bounded in suitable duration. See Bug 1112438. int64_t videoStart = video ? video->mTime : seekTime; int64_t audioStart = audio ? audio->mTime : seekTime; newCurrentTime = mAudioStartTime = std::min(audioStart, videoStart); } else { newCurrentTime = video ? video->mTime : seekTime; } mPlayDuration = newCurrentTime - mStartTime; mDecoder->StartProgressUpdates(); // Change state to DECODING or COMPLETED now. SeekingStopped will // call MediaDecoderStateMachine::Seek to reset our state to SEEKING // if we need to seek again. bool isLiveStream = mDecoder->GetResource()->IsLiveStream(); if (mPendingSeek.Exists()) { // A new seek target came in while we were processing the old one. No rest // for the seeking. DECODER_LOG("A new seek came along while we were finishing the old one - staying in SEEKING"); SetState(DECODER_STATE_SEEKING); } else if (GetMediaTime() == mEndTime && !isLiveStream) { // Seeked to end of media, move to COMPLETED state. Note we don't do // this if we're playing a live stream, since the end of media will advance // once we download more data! DECODER_LOG("Changed state from SEEKING (to %lld) to COMPLETED", seekTime); // Explicitly set our state so we don't decode further, and so // we report playback ended to the media element. SetState(DECODER_STATE_COMPLETED); DispatchDecodeTasksIfNeeded(); } else { DECODER_LOG("Changed state from SEEKING (to %lld) to DECODING", seekTime); StartDecoding(); } // Ensure timestamps are up to date. UpdatePlaybackPositionInternal(newCurrentTime); // Try to decode another frame to detect if we're at the end... DECODER_LOG("Seek completed, mCurrentFrameTime=%lld", mCurrentFrameTime); // Reset quick buffering status. This ensures that if we began the // seek while quick-buffering, we won't bypass quick buffering mode // if we need to buffer after the seek. mQuickBuffering = false; mCurrentSeek.Resolve(mState == DECODER_STATE_COMPLETED, __func__, TailDispatcher()); ScheduleStateMachine(); if (video) { ReentrantMonitorAutoExit exitMon(mDecoder->GetReentrantMonitor()); RenderVideoFrame(video, TimeStamp::Now()); nsCOMPtr event = NS_NewRunnableMethod(mDecoder, &MediaDecoder::Invalidate); TailDispatch(AbstractThread::MainThread(), event.forget()); } } class DecoderDisposer { public: NS_INLINE_DECL_THREADSAFE_REFCOUNTING(DecoderDisposer) DecoderDisposer(MediaDecoder* aDecoder, MediaDecoderStateMachine* aStateMachine) : mDecoder(aDecoder), mStateMachine(aStateMachine) {} void OnTaskQueueShutdown() { MOZ_ASSERT(NS_IsMainThread()); MOZ_ASSERT(mStateMachine); MOZ_ASSERT(mDecoder); mStateMachine->BreakCycles(); mDecoder->BreakCycles(); mStateMachine = nullptr; mDecoder = nullptr; } private: virtual ~DecoderDisposer() {} nsRefPtr mDecoder; nsRefPtr mStateMachine; }; void MediaDecoderStateMachine::ShutdownReader() { MOZ_ASSERT(OnDecodeTaskQueue()); mReader->Shutdown()->Then(TaskQueue(), __func__, this, &MediaDecoderStateMachine::FinishShutdown, &MediaDecoderStateMachine::FinishShutdown); } void MediaDecoderStateMachine::FinishShutdown() { MOZ_ASSERT(OnTaskQueue()); ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); // The reader's listeners hold references to the state machine, // creating a cycle which keeps the state machine and its shared // thread pools alive. So break it here. AudioQueue().ClearListeners(); VideoQueue().ClearListeners(); // Now that those threads are stopped, there's no possibility of // mPendingWakeDecoder being needed again. Revoke it. mPendingWakeDecoder = nullptr; MOZ_ASSERT(mState == DECODER_STATE_SHUTDOWN, "How did we escape from the shutdown state?"); // We must daisy-chain these events to destroy the decoder. We must // destroy the decoder on the main thread, but we can't destroy the // decoder while this thread holds the decoder monitor. We can't // dispatch an event to the main thread to destroy the decoder from // here, as the event may run before the dispatch returns, and we // hold the decoder monitor here. We also want to guarantee that the // state machine is destroyed on the main thread, and so the // event runner running this function (which holds a reference to the // state machine) needs to finish and be released in order to allow // that. So we dispatch an event to run after this event runner has // finished and released its monitor/references. That event then will // dispatch an event to the main thread to release the decoder and // state machine. DECODER_LOG("Shutting down state machine task queue"); RefPtr disposer = new DecoderDisposer(mDecoder, this); TaskQueue()->BeginShutdown()->Then(AbstractThread::MainThread(), __func__, disposer.get(), &DecoderDisposer::OnTaskQueueShutdown, &DecoderDisposer::OnTaskQueueShutdown, TailDispatcher()); } nsresult MediaDecoderStateMachine::RunStateMachine() { MOZ_ASSERT(OnTaskQueue()); ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); mDelayedScheduler.Reset(); // Must happen on state machine task queue. mDispatchedStateMachine = false; // If audio is being captured, stop the audio sink if it's running if (mAudioCaptured) { StopAudioThread(); } MediaResource* resource = mDecoder->GetResource(); NS_ENSURE_TRUE(resource, NS_ERROR_NULL_POINTER); switch (mState) { case DECODER_STATE_ERROR: { // Just wait for MediaDecoder::DecodeError to shut us down. return NS_OK; } case DECODER_STATE_SHUTDOWN: { mQueuedSeek.RejectIfExists(__func__, TailDispatcher()); mPendingSeek.RejectIfExists(__func__, TailDispatcher()); mCurrentSeek.RejectIfExists(__func__, TailDispatcher()); if (IsPlaying()) { StopPlayback(); } Reset(); // Put a task in the decode queue to shutdown the reader. // the queue to spin down. nsCOMPtr task = NS_NewRunnableMethod(this, &MediaDecoderStateMachine::ShutdownReader); TailDispatch(DecodeTaskQueue(), task.forget()); DECODER_LOG("Shutdown started"); return NS_OK; } case DECODER_STATE_DORMANT: { return NS_OK; } case DECODER_STATE_WAIT_FOR_CDM: case DECODER_STATE_WAIT_FOR_RESOURCES: { return NS_OK; } case DECODER_STATE_DECODING_NONE: { SetState(DECODER_STATE_DECODING_METADATA); ScheduleStateMachine(); return NS_OK; } case DECODER_STATE_DECODING_METADATA: { if (!mMetadataRequest.Exists()) { DECODER_LOG("Dispatching CallReadMetadata"); mMetadataRequest.Begin(ProxyMediaCall(DecodeTaskQueue(), mReader.get(), __func__, &MediaDecoderReader::CallReadMetadata, TailDispatcher()) ->RefableThen(TaskQueue(), __func__, this, &MediaDecoderStateMachine::OnMetadataRead, &MediaDecoderStateMachine::OnMetadataNotRead, TailDispatcher())); } return NS_OK; } case DECODER_STATE_DECODING_FIRSTFRAME: { // DECODER_STATE_DECODING_FIRSTFRAME will be started by OnMetadataRead. return NS_OK; } case DECODER_STATE_DECODING: { if (mDecoder->GetState() != MediaDecoder::PLAY_STATE_PLAYING && IsPlaying()) { // We're playing, but the element/decoder is in paused state. Stop // playing! StopPlayback(); } // Start playback if necessary so that the clock can be properly queried. MaybeStartPlayback(); AdvanceFrame(); NS_ASSERTION(mDecoder->GetState() != MediaDecoder::PLAY_STATE_PLAYING || IsStateMachineScheduled() || mPlaybackRate == 0.0, "Must have timer scheduled"); return NS_OK; } case DECODER_STATE_BUFFERING: { TimeStamp now = TimeStamp::Now(); NS_ASSERTION(!mBufferingStart.IsNull(), "Must know buffering start time."); // With buffering heuristics we will remain in the buffering state if // we've not decoded enough data to begin playback, or if we've not // downloaded a reasonable amount of data inside our buffering time. if (mReader->UseBufferingHeuristics()) { TimeDuration elapsed = now - mBufferingStart; bool isLiveStream = resource->IsLiveStream(); if ((isLiveStream || !mDecoder->CanPlayThrough()) && elapsed < TimeDuration::FromSeconds(mBufferingWait * mPlaybackRate) && (mQuickBuffering ? HasLowDecodedData(mQuickBufferingLowDataThresholdUsecs) : HasLowUndecodedData(mBufferingWait * USECS_PER_S)) && mDecoder->IsExpectingMoreData()) { DECODER_LOG("Buffering: wait %ds, timeout in %.3lfs %s", mBufferingWait, mBufferingWait - elapsed.ToSeconds(), (mQuickBuffering ? "(quick exit)" : "")); ScheduleStateMachineIn(USECS_PER_S); return NS_OK; } } else if (OutOfDecodedAudio() || OutOfDecodedVideo()) { MOZ_ASSERT(mReader->IsWaitForDataSupported(), "Don't yet have a strategy for non-heuristic + non-WaitForData"); DispatchDecodeTasksIfNeeded(); MOZ_ASSERT_IF(!mMinimizePreroll && OutOfDecodedAudio(), mAudioDataRequest.Exists() || mAudioWaitRequest.Exists()); MOZ_ASSERT_IF(!mMinimizePreroll && OutOfDecodedVideo(), mVideoDataRequest.Exists() || mVideoWaitRequest.Exists()); DECODER_LOG("In buffering mode, waiting to be notified: outOfAudio: %d, " "mAudioStatus: %s, outOfVideo: %d, mVideoStatus: %s", OutOfDecodedAudio(), AudioRequestStatus(), OutOfDecodedVideo(), VideoRequestStatus()); return NS_OK; } DECODER_LOG("Changed state from BUFFERING to DECODING"); DECODER_LOG("Buffered for %.3lfs", (now - mBufferingStart).ToSeconds()); StartDecoding(); // Notify to allow blocked decoder thread to continue mDecoder->GetReentrantMonitor().NotifyAll(); UpdateReadyState(); MaybeStartPlayback(); NS_ASSERTION(IsStateMachineScheduled(), "Must have timer scheduled"); return NS_OK; } case DECODER_STATE_SEEKING: { if (mPendingSeek.Exists()) { InitiateSeek(); } return NS_OK; } case DECODER_STATE_COMPLETED: { // Play the remaining media. We want to run AdvanceFrame() at least // once to ensure the current playback position is advanced to the // end of the media, and so that we update the readyState. if (VideoQueue().GetSize() > 0 || (HasAudio() && !mAudioCompleted) || (mAudioCaptured && !mDecoder->GetDecodedStream()->IsFinished())) { AdvanceFrame(); NS_ASSERTION(mDecoder->GetState() != MediaDecoder::PLAY_STATE_PLAYING || mPlaybackRate == 0 || IsStateMachineScheduled(), "Must have timer scheduled"); return NS_OK; } // StopPlayback in order to reset the IsPlaying() state so audio // is restarted correctly. StopPlayback(); if (mState != DECODER_STATE_COMPLETED) { // While we're presenting a frame we can change state. Whatever changed // our state should have scheduled another state machine run. NS_ASSERTION(IsStateMachineScheduled(), "Must have timer scheduled"); return NS_OK; } StopAudioThread(); if (mDecoder->GetState() == MediaDecoder::PLAY_STATE_PLAYING && !mSentPlaybackEndedEvent) { int64_t clockTime = std::max(mAudioEndTime, mVideoFrameEndTime); clockTime = std::max(int64_t(0), std::max(clockTime, mEndTime)); UpdatePlaybackPosition(clockTime); nsCOMPtr event = NS_NewRunnableMethod(mDecoder, &MediaDecoder::PlaybackEnded); TailDispatch(AbstractThread::MainThread(), event.forget()); mSentPlaybackEndedEvent = true; } return NS_OK; } } return NS_OK; } void MediaDecoderStateMachine::Reset() { MOZ_ASSERT(OnTaskQueue()); AssertCurrentThreadInMonitor(); DECODER_LOG("MediaDecoderStateMachine::Reset"); // We should be resetting because we're seeking, shutting down, or entering // dormant state. We could also be in the process of going dormant, and have // just switched to exiting dormant before we finished entering dormant, // hence the DECODING_NONE case below. MOZ_ASSERT(IsShutdown() || mState == DECODER_STATE_SEEKING || mState == DECODER_STATE_DORMANT || mState == DECODER_STATE_DECODING_NONE); // Stop the audio thread. Otherwise, AudioSink might be accessing AudioQueue // outside of the decoder monitor while we are clearing the queue and causes // crash for no samples to be popped. StopAudioThread(); mVideoFrameEndTime = -1; mDecodedVideoEndTime = -1; mAudioStartTime = -1; mAudioEndTime = -1; mDecodedAudioEndTime = -1; mAudioCompleted = false; AudioQueue().Reset(); VideoQueue().Reset(); mFirstVideoFrameAfterSeek = nullptr; mDropAudioUntilNextDiscontinuity = true; mDropVideoUntilNextDiscontinuity = true; mDecodeToSeekTarget = false; mMetadataRequest.DisconnectIfExists(); mAudioDataRequest.DisconnectIfExists(); mAudioWaitRequest.DisconnectIfExists(); mVideoDataRequest.DisconnectIfExists(); mVideoWaitRequest.DisconnectIfExists(); mSeekRequest.DisconnectIfExists(); nsCOMPtr resetTask = NS_NewRunnableMethod(mReader, &MediaDecoderReader::ResetDecode); TailDispatch(DecodeTaskQueue(), resetTask.forget()); } void MediaDecoderStateMachine::RenderVideoFrame(VideoData* aData, TimeStamp aTarget) { MOZ_ASSERT(OnTaskQueue()); mDecoder->GetReentrantMonitor().AssertNotCurrentThreadIn(); if (aData->mDuplicate) { return; } VERBOSE_LOG("playing video frame %lld (queued=%i, state-machine=%i, decoder-queued=%i)", aData->mTime, VideoQueue().GetSize() + mReader->SizeOfVideoQueueInFrames(), VideoQueue().GetSize(), mReader->SizeOfVideoQueueInFrames()); VideoFrameContainer* container = mDecoder->GetVideoFrameContainer(); if (container) { if (aData->mImage && !aData->mImage->IsValid()) { MediaDecoder::FrameStatistics& frameStats = mDecoder->GetFrameStatistics(); frameStats.NotifyCorruptFrame(); // If more than 10% of the last 30 frames have been corrupted, then try disabling // hardware acceleration. We use 10 as the corrupt value because RollingMean<> // only supports integer types. mCorruptFrames.insert(10); if (!mDisabledHardwareAcceleration && mReader->VideoIsHardwareAccelerated() && frameStats.GetPresentedFrames() > 30 && mCorruptFrames.mean() >= 1 /* 10% */) { nsCOMPtr task = NS_NewRunnableMethod(mReader, &MediaDecoderReader::DisableHardwareAcceleration); TailDispatch(DecodeTaskQueue(), task.forget()); mDisabledHardwareAcceleration = true; } } else { mCorruptFrames.insert(0); } container->SetCurrentFrame(aData->mDisplay, aData->mImage, aTarget); MOZ_ASSERT(container->GetFrameDelay() >= 0 || IsRealTime()); } } void MediaDecoderStateMachine::ResyncAudioClock() { AssertCurrentThreadInMonitor(); if (IsPlaying()) { SetPlayStartTime(TimeStamp::Now()); mPlayDuration = GetAudioClock() - mStartTime; } } int64_t MediaDecoderStateMachine::GetAudioClock() const { // We must hold the decoder monitor while using the audio stream off the // audio sink to ensure that it doesn't get destroyed on the audio sink // while we're using it. AssertCurrentThreadInMonitor(); MOZ_ASSERT(HasAudio() && !mAudioCompleted); return mAudioStartTime + (mAudioSink ? mAudioSink->GetPosition() : 0); } int64_t MediaDecoderStateMachine::GetVideoStreamPosition() const { AssertCurrentThreadInMonitor(); if (!IsPlaying()) { return mPlayDuration + mStartTime; } // Time elapsed since we started playing. int64_t delta = DurationToUsecs(TimeStamp::Now() - mPlayStartTime); // Take playback rate into account. delta *= mPlaybackRate; return mStartTime + mPlayDuration + delta; } int64_t MediaDecoderStateMachine::GetClock() const { AssertCurrentThreadInMonitor(); // Determine the clock time. If we've got audio, and we've not reached // the end of the audio, use the audio clock. However if we've finished // audio, or don't have audio, use the system clock. If our output is being // fed to a MediaStream, use that stream as the source of the clock. int64_t clock_time = -1; if (!IsPlaying()) { clock_time = mPlayDuration + mStartTime; } else { if (mAudioCaptured) { clock_time = mStartTime + mDecoder->GetDecodedStream()->GetClock(); } else if (HasAudio() && !mAudioCompleted) { clock_time = GetAudioClock(); } else { // Audio is disabled on this system. Sync to the system clock. clock_time = GetVideoStreamPosition(); } // Ensure the clock can never go backwards. // Note we allow clock going backwards in capture mode during seeking. NS_ASSERTION(GetMediaTime() <= clock_time || mPlaybackRate <= 0 || (mAudioCaptured && mState == DECODER_STATE_SEEKING), "Clock should go forwards."); } return clock_time; } void MediaDecoderStateMachine::AdvanceFrame() { MOZ_ASSERT(OnTaskQueue()); AssertCurrentThreadInMonitor(); NS_ASSERTION(!HasAudio() || mAudioStartTime != -1, "Should know audio start time if we have audio."); if (mDecoder->GetState() != MediaDecoder::PLAY_STATE_PLAYING) { return; } // If playbackRate is 0.0, we should stop the progress, but not be in paused // state, per spec. if (mPlaybackRate == 0.0) { return; } if (mAudioCaptured) { SendStreamData(); } const int64_t clock_time = GetClock(); TimeStamp nowTime = TimeStamp::Now(); // Skip frames up to the frame at the playback position, and figure out // the time remaining until it's time to display the next frame. int64_t remainingTime = AUDIO_DURATION_USECS; NS_ASSERTION(clock_time >= mStartTime, "Should have positive clock time."); nsRefPtr currentFrame; if (VideoQueue().GetSize() > 0) { VideoData* frame = VideoQueue().PeekFront(); #ifdef PR_LOGGING int32_t droppedFrames = 0; #endif while (IsRealTime() || clock_time >= frame->mTime) { mVideoFrameEndTime = frame->GetEndTime(); if (currentFrame) { mDecoder->NotifyDecodedFrames(0, 0, 1); #ifdef PR_LOGGING VERBOSE_LOG("discarding video frame mTime=%lld clock_time=%lld (%d so far)", currentFrame->mTime, clock_time, ++droppedFrames); #endif } currentFrame = frame; nsRefPtr releaseMe = VideoQueue().PopFront(); // Notify the decode thread that the video queue's buffers may have // free'd up space for more frames. mDecoder->GetReentrantMonitor().NotifyAll(); OnPlaybackOffsetUpdate(frame->mOffset); if (VideoQueue().GetSize() == 0) break; frame = VideoQueue().PeekFront(); } // Current frame has already been presented, wait until it's time to // present the next frame. if (frame && !currentFrame) { int64_t now = IsPlaying() ? clock_time : mStartTime + mPlayDuration; remainingTime = frame->mTime - now; } } // Check to see if we don't have enough data to play up to the next frame. // If we don't, switch to buffering mode. if (mState == DECODER_STATE_DECODING && mDecoder->GetState() == MediaDecoder::PLAY_STATE_PLAYING && mDecoder->IsExpectingMoreData()) { bool shouldBuffer; if (mReader->UseBufferingHeuristics()) { shouldBuffer = HasLowDecodedData(remainingTime + EXHAUSTED_DATA_MARGIN_USECS) && (JustExitedQuickBuffering() || HasLowUndecodedData()); } else { MOZ_ASSERT(mReader->IsWaitForDataSupported()); shouldBuffer = (OutOfDecodedAudio() && mAudioWaitRequest.Exists()) || (OutOfDecodedVideo() && mVideoWaitRequest.Exists()); } if (shouldBuffer) { if (currentFrame) { VideoQueue().PushFront(currentFrame); } StartBuffering(); // Don't go straight back to the state machine loop since that might // cause us to start decoding again and we could flip-flop between // decoding and quick-buffering. ScheduleStateMachineIn(USECS_PER_S); return; } } // We've got enough data to keep playing until at least the next frame. // Start playing now if need be. if ((mFragmentEndTime >= 0 && clock_time < mFragmentEndTime) || mFragmentEndTime < 0) { MaybeStartPlayback(); } // Cap the current time to the larger of the audio and video end time. // This ensures that if we're running off the system clock, we don't // advance the clock to after the media end time. if (mVideoFrameEndTime != -1 || mAudioEndTime != -1) { // These will be non -1 if we've displayed a video frame, or played an audio frame. int64_t t = std::min(clock_time, std::max(mVideoFrameEndTime, mAudioEndTime)); // FIXME: Bug 1091422 - chained ogg files hit this assertion. //MOZ_ASSERT(t >= GetMediaTime()); if (t > GetMediaTime()) { UpdatePlaybackPosition(t); } } // Note we have to update playback position before releasing the monitor. // Otherwise, MediaDecoder::AddOutputStream could kick in when we are outside // the monitor and get a staled value from GetCurrentTimeUs() which hits the // assertion in GetClock(). if (currentFrame) { // Decode one frame and display it. int64_t delta = currentFrame->mTime - clock_time; TimeStamp presTime = nowTime + TimeDuration::FromMicroseconds(delta / mPlaybackRate); NS_ASSERTION(currentFrame->mTime >= mStartTime, "Should have positive frame time"); // Filter out invalid frames by checking the frame time. FrameTime could be // zero if it's a initial frame. int64_t frameTime = currentFrame->mTime - mStartTime; if (frameTime > 0 || (frameTime == 0 && mPlayDuration == 0) || IsRealTime()) { ReentrantMonitorAutoExit exitMon(mDecoder->GetReentrantMonitor()); // If we have video, we want to increment the clock in steps of the frame // duration. RenderVideoFrame(currentFrame, presTime); } // If we're no longer playing after dropping and reacquiring the lock, // playback must've been stopped on the decode thread (by a seek, for // example). In that case, the current frame is probably out of date. if (!IsPlaying()) { ScheduleStateMachine(); return; } MediaDecoder::FrameStatistics& frameStats = mDecoder->GetFrameStatistics(); frameStats.NotifyPresentedFrame(); remainingTime = currentFrame->GetEndTime() - clock_time; currentFrame = nullptr; } // If the number of audio/video frames queued has changed, either by // this function popping and playing a video frame, or by the audio // thread popping and playing an audio frame, we may need to update our // ready state. Post an update to do so. UpdateReadyState(); int64_t delay = remainingTime / mPlaybackRate; if (delay > 0) { ScheduleStateMachineIn(delay); } else { ScheduleStateMachine(); } } nsresult MediaDecoderStateMachine::DropVideoUpToSeekTarget(VideoData* aSample) { nsRefPtr video(aSample); MOZ_ASSERT(video); DECODER_LOG("DropVideoUpToSeekTarget() frame [%lld, %lld] dup=%d", video->mTime, video->GetEndTime(), video->mDuplicate); MOZ_ASSERT(mCurrentSeek.Exists()); const int64_t target = mCurrentSeek.mTarget.mTime; // Duplicate handling: if we're dropping frames up the seek target, we must // be wary of Theora duplicate frames. They don't have an image, so if the // target frame is in a run of duplicates, we won't have an image to draw // after the seek. So store the last frame encountered while dropping, and // copy its Image forward onto duplicate frames, so that every frame has // an Image. if (video->mDuplicate && mFirstVideoFrameAfterSeek && !mFirstVideoFrameAfterSeek->mDuplicate) { nsRefPtr temp = VideoData::ShallowCopyUpdateTimestampAndDuration(mFirstVideoFrameAfterSeek, video->mTime, video->mDuration); video = temp; } // If the frame end time is less than the seek target, we won't want // to display this frame after the seek, so discard it. if (target >= video->GetEndTime()) { DECODER_LOG("DropVideoUpToSeekTarget() pop video frame [%lld, %lld] target=%lld", video->mTime, video->GetEndTime(), target); mFirstVideoFrameAfterSeek = video; } else { if (target >= video->mTime && video->GetEndTime() >= target) { // The seek target lies inside this frame's time slice. Adjust the frame's // start time to match the seek target. We do this by replacing the // first frame with a shallow copy which has the new timestamp. nsRefPtr temp = VideoData::ShallowCopyUpdateTimestamp(video, target); video = temp; } mFirstVideoFrameAfterSeek = nullptr; DECODER_LOG("DropVideoUpToSeekTarget() found video frame [%lld, %lld] containing target=%lld", video->mTime, video->GetEndTime(), target); VideoQueue().PushFront(video); } return NS_OK; } nsresult MediaDecoderStateMachine::DropAudioUpToSeekTarget(AudioData* aSample) { nsRefPtr audio(aSample); MOZ_ASSERT(audio && mCurrentSeek.Exists() && mCurrentSeek.mTarget.mType == SeekTarget::Accurate); CheckedInt64 startFrame = UsecsToFrames(audio->mTime, mInfo.mAudio.mRate); CheckedInt64 targetFrame = UsecsToFrames(mCurrentSeek.mTarget.mTime, mInfo.mAudio.mRate); if (!startFrame.isValid() || !targetFrame.isValid()) { return NS_ERROR_FAILURE; } if (startFrame.value() + audio->mFrames <= targetFrame.value()) { // Our seek target lies after the frames in this AudioData. Don't // push it onto the audio queue, and keep decoding forwards. return NS_OK; } if (startFrame.value() > targetFrame.value()) { // The seek target doesn't lie in the audio block just after the last // audio frames we've seen which were before the seek target. This // could have been the first audio data we've seen after seek, i.e. the // seek terminated after the seek target in the audio stream. Just // abort the audio decode-to-target, the state machine will play // silence to cover the gap. Typically this happens in poorly muxed // files. DECODER_WARN("Audio not synced after seek, maybe a poorly muxed file?"); AudioQueue().Push(audio); return NS_OK; } // The seek target lies somewhere in this AudioData's frames, strip off // any frames which lie before the seek target, so we'll begin playback // exactly at the seek target. NS_ASSERTION(targetFrame.value() >= startFrame.value(), "Target must at or be after data start."); NS_ASSERTION(targetFrame.value() < startFrame.value() + audio->mFrames, "Data must end after target."); int64_t framesToPrune = targetFrame.value() - startFrame.value(); if (framesToPrune > audio->mFrames) { // We've messed up somehow. Don't try to trim frames, the |frames| // variable below will overflow. DECODER_WARN("Can't prune more frames that we have!"); return NS_ERROR_FAILURE; } uint32_t frames = audio->mFrames - static_cast(framesToPrune); uint32_t channels = audio->mChannels; nsAutoArrayPtr audioData(new AudioDataValue[frames * channels]); memcpy(audioData.get(), audio->mAudioData.get() + (framesToPrune * channels), frames * channels * sizeof(AudioDataValue)); CheckedInt64 duration = FramesToUsecs(frames, mInfo.mAudio.mRate); if (!duration.isValid()) { return NS_ERROR_FAILURE; } nsRefPtr data(new AudioData(audio->mOffset, mCurrentSeek.mTarget.mTime, duration.value(), frames, audioData.forget(), channels, audio->mRate)); AudioQueue().PushFront(data); return NS_OK; } void MediaDecoderStateMachine::SetStartTime(int64_t aStartTimeUsecs) { AssertCurrentThreadInMonitor(); DECODER_LOG("SetStartTime(%lld)", aStartTimeUsecs); mStartTime = 0; if (aStartTimeUsecs != 0) { mStartTime = aStartTimeUsecs; if (mGotDurationFromMetaData && GetEndTime() != INT64_MAX) { NS_ASSERTION(mEndTime != -1, "We should have mEndTime as supplied duration here"); // We were specified a duration from a Content-Duration HTTP header. // Adjust mEndTime so that mEndTime-mStartTime matches the specified // duration. mEndTime = mStartTime + mEndTime; } } // Pass along this immutable value to the reader so that it can make // calculations independently of the state machine. mReader->SetStartTime(mStartTime); // Set the audio start time to be start of media. If this lies before the // first actual audio frame we have, we'll inject silence during playback // to ensure the audio starts at the correct time. mAudioStartTime = mStartTime; DECODER_LOG("Set media start time to %lld", mStartTime); } void MediaDecoderStateMachine::UpdateReadyState() { AssertCurrentThreadInMonitor(); MediaDecoderOwner::NextFrameStatus nextFrameStatus = GetNextFrameStatus(); // FIXME: This optimization could result in inconsistent next frame status // between the decoder and state machine when GetNextFrameStatus() is called // by the decoder without updating mLastFrameStatus. // Note not to regress bug 882027 when fixing this bug. if (nextFrameStatus == mLastFrameStatus) { return; } mLastFrameStatus = nextFrameStatus; /* This is a bit tricky. MediaDecoder::UpdateReadyStateForData will run on * the main thread and re-evaluate GetNextFrameStatus there, passing it to * HTMLMediaElement::UpdateReadyStateForData. It doesn't use the value of * GetNextFrameStatus we computed here, because what we're computing here * could be stale by the time MediaDecoder::UpdateReadyStateForData runs. * We only compute GetNextFrameStatus here to avoid posting runnables to the main * thread unnecessarily. */ nsCOMPtr event; event = NS_NewRunnableMethod(mDecoder, &MediaDecoder::UpdateReadyStateForData); AbstractThread::MainThread()->MaybeTailDispatch(event.forget()); } bool MediaDecoderStateMachine::JustExitedQuickBuffering() { return !mDecodeStartTime.IsNull() && mQuickBuffering && (TimeStamp::Now() - mDecodeStartTime) < TimeDuration::FromMicroseconds(QUICK_BUFFER_THRESHOLD_USECS); } void MediaDecoderStateMachine::StartBuffering() { MOZ_ASSERT(OnTaskQueue()); ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); if (mState != DECODER_STATE_DECODING) { // We only move into BUFFERING state if we're actually decoding. // If we're currently doing something else, we don't need to buffer, // and more importantly, we shouldn't overwrite mState to interrupt // the current operation, as that could leave us in an inconsistent // state! return; } if (IsPlaying()) { StopPlayback(); } TimeDuration decodeDuration = TimeStamp::Now() - mDecodeStartTime; // Go into quick buffering mode provided we've not just left buffering using // a "quick exit". This stops us flip-flopping between playing and buffering // when the download speed is similar to the decode speed. mQuickBuffering = !JustExitedQuickBuffering() && decodeDuration < UsecsToDuration(QUICK_BUFFER_THRESHOLD_USECS); mBufferingStart = TimeStamp::Now(); // We need to tell the element that buffering has started. // We can't just directly send an asynchronous runnable that // eventually fires the "waiting" event. The problem is that // there might be pending main-thread events, such as "data // received" notifications, that mean we're not actually still // buffering by the time this runnable executes. So instead // we just trigger UpdateReadyStateForData; when it runs, it // will check the current state and decide whether to tell // the element we're buffering or not. SetState(DECODER_STATE_BUFFERING); UpdateReadyState(); DECODER_LOG("Changed state from DECODING to BUFFERING, decoded for %.3lfs", decodeDuration.ToSeconds()); #ifdef PR_LOGGING MediaDecoder::Statistics stats = mDecoder->GetStatistics(); DECODER_LOG("Playback rate: %.1lfKB/s%s download rate: %.1lfKB/s%s", stats.mPlaybackRate/1024, stats.mPlaybackRateReliable ? "" : " (unreliable)", stats.mDownloadRate/1024, stats.mDownloadRateReliable ? "" : " (unreliable)"); #endif } void MediaDecoderStateMachine::SetPlayStartTime(const TimeStamp& aTimeStamp) { AssertCurrentThreadInMonitor(); mPlayStartTime = aTimeStamp; if (!mAudioSink) { return; } if (!mPlayStartTime.IsNull()) { mAudioSink->StartPlayback(); } else { mAudioSink->StopPlayback(); } } void MediaDecoderStateMachine::ScheduleStateMachineWithLockAndWakeDecoder() { ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); DispatchAudioDecodeTaskIfNeeded(); DispatchVideoDecodeTaskIfNeeded(); } void MediaDecoderStateMachine::ScheduleStateMachine() { AssertCurrentThreadInMonitor(); if (mState == DECODER_STATE_SHUTDOWN) { NS_WARNING("Refusing to schedule shutdown state machine"); return; } if (mDispatchedStateMachine) { return; } mDispatchedStateMachine = true; nsCOMPtr task = NS_NewRunnableMethod(this, &MediaDecoderStateMachine::RunStateMachine); TaskQueue()->MaybeTailDispatch(task.forget()); } void MediaDecoderStateMachine::ScheduleStateMachineIn(int64_t aMicroseconds) { AssertCurrentThreadInMonitor(); MOZ_ASSERT(OnTaskQueue()); // mDelayedScheduler.Ensure() may Disconnect() // the promise, which must happen on the state // machine task queue. MOZ_ASSERT(aMicroseconds > 0); if (mState == DECODER_STATE_SHUTDOWN) { NS_WARNING("Refusing to schedule shutdown state machine"); return; } if (mDispatchedStateMachine) { return; } // Real-time weirdness. if (IsRealTime()) { aMicroseconds = std::min(aMicroseconds, int64_t(40000)); } TimeStamp now = TimeStamp::Now(); TimeStamp target = now + TimeDuration::FromMicroseconds(aMicroseconds); SAMPLE_LOG("Scheduling state machine for %lf ms from now", (target - now).ToMilliseconds()); mDelayedScheduler.Ensure(target); } bool MediaDecoderStateMachine::OnDecodeTaskQueue() const { return !DecodeTaskQueue() || DecodeTaskQueue()->IsCurrentThreadIn(); } bool MediaDecoderStateMachine::OnTaskQueue() const { return TaskQueue()->IsCurrentThreadIn(); } bool MediaDecoderStateMachine::IsStateMachineScheduled() const { return mDispatchedStateMachine || mDelayedScheduler.IsScheduled(); } void MediaDecoderStateMachine::SetPlaybackRate(double aPlaybackRate) { NS_ASSERTION(NS_IsMainThread(), "Should be on main thread."); NS_ASSERTION(aPlaybackRate != 0, "PlaybackRate == 0 should be handled before this function."); ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); if (mPlaybackRate == aPlaybackRate) { return; } // AudioStream will handle playback rate change when we have audio. // Do nothing while we are not playing. Change in playback rate will // take effect next time we start playing again. if (!HasAudio() && IsPlaying()) { // Remember how much time we've spent in playing the media // for playback rate will change from now on. mPlayDuration = GetVideoStreamPosition() - mStartTime; SetPlayStartTime(TimeStamp::Now()); } mPlaybackRate = aPlaybackRate; if (mAudioSink) { mAudioSink->SetPlaybackRate(mPlaybackRate); } } void MediaDecoderStateMachine::SetPreservesPitch(bool aPreservesPitch) { NS_ASSERTION(NS_IsMainThread(), "Should be on main thread."); ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); mPreservesPitch = aPreservesPitch; if (mAudioSink) { mAudioSink->SetPreservesPitch(mPreservesPitch); } } void MediaDecoderStateMachine::SetMinimizePrerollUntilPlaybackStarts() { AssertCurrentThreadInMonitor(); DECODER_LOG("SetMinimizePrerollUntilPlaybackStarts()"); mMinimizePreroll = true; } bool MediaDecoderStateMachine::IsShutdown() { AssertCurrentThreadInMonitor(); return mState == DECODER_STATE_ERROR || mState == DECODER_STATE_SHUTDOWN; } void MediaDecoderStateMachine::QueueMetadata(int64_t aPublishTime, nsAutoPtr aInfo, nsAutoPtr aTags) { MOZ_ASSERT(OnDecodeTaskQueue()); AssertCurrentThreadInMonitor(); TimedMetadata* metadata = new TimedMetadata; metadata->mPublishTime = aPublishTime; metadata->mInfo = aInfo.forget(); metadata->mTags = aTags.forget(); mMetadataManager.QueueMetadata(metadata); } void MediaDecoderStateMachine::OnAudioEndTimeUpdate(int64_t aAudioEndTime) { MOZ_ASSERT(OnTaskQueue()); ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); MOZ_ASSERT(aAudioEndTime >= mAudioEndTime); mAudioEndTime = aAudioEndTime; } void MediaDecoderStateMachine::OnPlaybackOffsetUpdate(int64_t aPlaybackOffset) { MOZ_ASSERT(OnTaskQueue()); ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); mDecoder->UpdatePlaybackOffset(aPlaybackOffset); } void MediaDecoderStateMachine::OnAudioSinkComplete() { MOZ_ASSERT(OnTaskQueue()); ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); if (mAudioCaptured) { return; } ResyncAudioClock(); mAudioCompleted = true; UpdateReadyState(); // Kick the decode thread; it may be sleeping waiting for this to finish. mDecoder->GetReentrantMonitor().NotifyAll(); } void MediaDecoderStateMachine::OnAudioSinkError() { MOZ_ASSERT(OnTaskQueue()); ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); // AudioSink not used with captured streams, so ignore errors in this case. if (mAudioCaptured) { return; } ResyncAudioClock(); mAudioCompleted = true; // Make the best effort to continue playback when there is video. if (HasVideo()) { return; } // Otherwise notify media decoder/element about this error for it makes // no sense to play an audio-only file without sound output. DecodeError(); } uint32_t MediaDecoderStateMachine::GetAmpleVideoFrames() const { AssertCurrentThreadInMonitor(); return (mReader->IsAsync() && mReader->VideoIsHardwareAccelerated()) ? std::max(sVideoQueueHWAccelSize, MIN_VIDEO_QUEUE_SIZE) : std::max(sVideoQueueDefaultSize, MIN_VIDEO_QUEUE_SIZE); } } // namespace mozilla // avoid redefined macro in unified build #undef DECODER_LOG #undef VERBOSE_LOG #undef DECODER_WARN #undef DECODER_WARN_HELPER #undef NS_DispatchToMainThread