From c949841839e82c1600f679296bbe736b9de0554e Mon Sep 17 00:00:00 2001 From: Bryce Van Dyk Date: Fri, 12 Jul 2019 13:40:25 +0000 Subject: [PATCH] Bug 1014393 - Separate MediaEncoders encode and mux steps. r=pehrsons Separating the encode and mux steps allows for better control over interleaving audio and video data. If encode and mux are done in a single step it's possible to mux large amounts of audio or video data which should have been interleaved with the other data type to give correctly ordered time stamps in the target container. MozReview-Commit-ID: CBYemrkpyF5 Differential Revision: https://phabricator.services.mozilla.com/D35383 --HG-- extra : moz-landing-system : lando --- dom/media/encoder/MediaEncoder.cpp | 126 +++++++++++++++++++---------- dom/media/encoder/MediaEncoder.h | 12 ++- 2 files changed, 95 insertions(+), 43 deletions(-) diff --git a/dom/media/encoder/MediaEncoder.cpp b/dom/media/encoder/MediaEncoder.cpp index dae45e4e5e14..a1a7455b37c5 100644 --- a/dom/media/encoder/MediaEncoder.cpp +++ b/dom/media/encoder/MediaEncoder.cpp @@ -821,28 +821,15 @@ nsresult MediaEncoder::GetEncodedData( nsresult rv; LOG(LogLevel::Verbose, ("GetEncodedData TimeStamp = %f", GetEncodeTimeStamp())); - EncodedFrameContainer encodedData; - if (mVideoEncoder) { - // We're most likely to actually wait for a video frame, so do that first - // to minimize capture offset/lipsync issues. - rv = WriteEncodedDataToMuxer(mVideoEncoder); - LOG(LogLevel::Verbose, - ("Video encoded TimeStamp = %f", GetEncodeTimeStamp())); - if (NS_FAILED(rv)) { - LOG(LogLevel::Warning, ("Failed to write encoded video data to muxer")); - return rv; - } + rv = EncodeData(); + if (NS_FAILED(rv)) { + return rv; } - if (mAudioEncoder) { - rv = WriteEncodedDataToMuxer(mAudioEncoder); - LOG(LogLevel::Verbose, - ("Audio encoded TimeStamp = %f", GetEncodeTimeStamp())); - if (NS_FAILED(rv)) { - LOG(LogLevel::Warning, ("Failed to write encoded audio data to muxer")); - return rv; - } + rv = WriteEncodedDataToMuxer(); + if (NS_FAILED(rv)) { + return rv; } // In audio only or video only case, let unavailable track's flag to be @@ -904,37 +891,94 @@ void MediaEncoder::Shutdown() { } } -nsresult MediaEncoder::WriteEncodedDataToMuxer(TrackEncoder* aTrackEncoder) { +nsresult MediaEncoder::EncodeData() { + AUTO_PROFILER_LABEL("MediaEncoder::EncodeData", OTHER); + + MOZ_ASSERT(mEncoderThread->IsCurrentThreadIn()); + + if (!mVideoEncoder && !mAudioEncoder) { + MOZ_ASSERT_UNREACHABLE("Must have atleast one encoder"); + return NS_ERROR_UNEXPECTED; + } + + if (mVideoEncoder && !mVideoEncoder->IsEncodingComplete()) { + EncodedFrameContainer encodedVideoData; + nsresult rv = mVideoEncoder->GetEncodedTrack(encodedVideoData); + if (NS_FAILED(rv)) { + // Encoding might be canceled. + LOG(LogLevel::Error, ("Failed to get encoded data from video encoder.")); + return rv; + } + for (const RefPtr& frame : + encodedVideoData.GetEncodedFrames()) { + mEncodedVideoFrames.AppendElement(frame); + } + } + + if (mAudioEncoder && !mAudioEncoder->IsEncodingComplete()) { + EncodedFrameContainer encodedAudioData; + nsresult rv = mAudioEncoder->GetEncodedTrack(encodedAudioData); + if (NS_FAILED(rv)) { + // Encoding might be canceled. + LOG(LogLevel::Error, ("Failed to get encoded data from audio encoder.")); + return rv; + } + for (const RefPtr& frame : + encodedAudioData.GetEncodedFrames()) { + mEncodedAudioFrames.AppendElement(frame); + } + } + + return NS_OK; +} + +nsresult MediaEncoder::WriteEncodedDataToMuxer() { AUTO_PROFILER_LABEL("MediaEncoder::WriteEncodedDataToMuxer", OTHER); MOZ_ASSERT(mEncoderThread->IsCurrentThreadIn()); - if (!aTrackEncoder) { - NS_ERROR("No track encoder to get data from"); - return NS_ERROR_FAILURE; + if (!mVideoEncoder && !mAudioEncoder) { + MOZ_ASSERT_UNREACHABLE("Must have atleast one encoder"); + return NS_ERROR_UNEXPECTED; } - if (aTrackEncoder->IsEncodingComplete()) { - return NS_OK; + if (mVideoEncoder) { + EncodedFrameContainer encodedVideoData; + for (const RefPtr& frame : mEncodedVideoFrames) { + encodedVideoData.AppendEncodedFrame(frame); + } + mEncodedVideoFrames.Clear(); + + nsresult rv = mWriter->WriteEncodedTrack( + encodedVideoData, mVideoEncoder->IsEncodingComplete() + ? ContainerWriter::END_OF_STREAM + : 0); + if (NS_FAILED(rv)) { + LOG(LogLevel::Error, + ("Failed to write encoded video track to the muxer.")); + return rv; + } } - EncodedFrameContainer encodedData; - nsresult rv = aTrackEncoder->GetEncodedTrack(encodedData); - if (NS_FAILED(rv)) { - // Encoding might be canceled. - LOG(LogLevel::Error, ("Failed to get encoded data from encoder.")); - SetError(); - return rv; + if (mAudioEncoder) { + EncodedFrameContainer encodedAudioData; + for (const RefPtr& frame : mEncodedAudioFrames) { + encodedAudioData.AppendEncodedFrame(frame); + } + mEncodedAudioFrames.Clear(); + + nsresult rv = mWriter->WriteEncodedTrack( + encodedAudioData, mAudioEncoder->IsEncodingComplete() + ? ContainerWriter::END_OF_STREAM + : 0); + if (NS_FAILED(rv)) { + LOG(LogLevel::Error, + ("Failed to write encoded audio track to the muxer.")); + return rv; + } } - rv = mWriter->WriteEncodedTrack( - encodedData, - aTrackEncoder->IsEncodingComplete() ? ContainerWriter::END_OF_STREAM : 0); - if (NS_FAILED(rv)) { - LOG(LogLevel::Error, - ("Failed to write encoded track to the media container.")); - SetError(); - } - return rv; + + return NS_OK; } nsresult MediaEncoder::CopyMetadataToMuxer(TrackEncoder* aTrackEncoder) { diff --git a/dom/media/encoder/MediaEncoder.h b/dom/media/encoder/MediaEncoder.h index 0e457f09ca2b..17b7c486b224 100644 --- a/dom/media/encoder/MediaEncoder.h +++ b/dom/media/encoder/MediaEncoder.h @@ -253,10 +253,12 @@ class MediaEncoder { */ void SetError(); - // Get encoded data from trackEncoder and write to muxer - nsresult WriteEncodedDataToMuxer(TrackEncoder* aTrackEncoder); // Get metadata from trackEncoder and copy to muxer nsresult CopyMetadataToMuxer(TrackEncoder* aTrackEncoder); + // Process data pending in encoder(s) + nsresult EncodeData(); + // Write pending encoded data to muxer + nsresult WriteEncodedDataToMuxer(); const RefPtr mEncoderThread; const RefPtr mDriftCompensator; @@ -284,6 +286,12 @@ class MediaEncoder { // A video track that we are encoding. Will be null if the input stream // doesn't contain video on start() or if the input is an AudioNode. RefPtr mVideoTrack; + + // Audio frames that have been encoded and are pending write to the muxer + nsTArray> mEncodedAudioFrames; + // Video frames that have been encoded and are pending write to the muxer + nsTArray> mEncodedVideoFrames; + TimeStamp mStartTime; nsString mMIMEType; bool mInitialized;