From f64409a83bfd07aa8e117cd067ee1817a67f85cf Mon Sep 17 00:00:00 2001 From: Randy Lin Date: Tue, 10 Dec 2013 15:24:51 +0800 Subject: [PATCH] Bug 879669 - Part 2: Let Encoder can support various mimeType. r=roc --- content/media/encoder/ContainerWriter.h | 14 +- content/media/encoder/EncodedFrameContainer.h | 2 + content/media/encoder/MediaEncoder.cpp | 269 +++++++++++------- content/media/encoder/MediaEncoder.h | 21 +- content/media/ogg/OggWriter.cpp | 4 +- content/media/ogg/OggWriter.h | 1 - 6 files changed, 206 insertions(+), 105 deletions(-) diff --git a/content/media/encoder/ContainerWriter.h b/content/media/encoder/ContainerWriter.h index c6fb5388623c..c8791ae9d6d1 100644 --- a/content/media/encoder/ContainerWriter.h +++ b/content/media/encoder/ContainerWriter.h @@ -19,9 +19,14 @@ class ContainerWriter { public: ContainerWriter() : mInitialized(false) + , mIsWritingComplete(false) {} virtual ~ContainerWriter() {} - + // Mapping to DOMLocalMediaStream::TrackTypeHints + enum { + HAS_AUDIO = 1 << 0, + HAS_VIDEO = 1 << 1, + }; enum { END_OF_STREAM = 1 << 0 }; @@ -44,6 +49,11 @@ public: */ virtual nsresult SetMetadata(TrackMetadataBase* aMetadata) = 0; + /** + * Indicate if the writer has finished to output data + */ + virtual bool IsWritingComplete() { return mIsWritingComplete; } + enum { FLUSH_NEEDED = 1 << 0, GET_HEADER = 1 << 1 @@ -59,9 +69,9 @@ public: */ virtual nsresult GetContainerData(nsTArray >* aOutputBufs, uint32_t aFlags = 0) = 0; - protected: bool mInitialized; + bool mIsWritingComplete; }; } #endif diff --git a/content/media/encoder/EncodedFrameContainer.h b/content/media/encoder/EncodedFrameContainer.h index 6b665f8c305a..4335464ab88d 100644 --- a/content/media/encoder/EncodedFrameContainer.h +++ b/content/media/encoder/EncodedFrameContainer.h @@ -52,6 +52,8 @@ public: P_FRAME, // predicted frame B_FRAME, // bidirectionally predicted frame AUDIO_FRAME, // audio frame + AAC_CSD, // AAC codec specific data + AVC_CSD, // AVC codec specific data UNKNOW // FrameType not set }; const nsTArray& GetFrameData() const diff --git a/content/media/encoder/MediaEncoder.cpp b/content/media/encoder/MediaEncoder.cpp index 7361bae39b3f..60931f345217 100644 --- a/content/media/encoder/MediaEncoder.cpp +++ b/content/media/encoder/MediaEncoder.cpp @@ -5,6 +5,8 @@ #include "MediaEncoder.h" #include "MediaDecoder.h" #include "nsIPrincipal.h" +#include "nsMimeTypes.h" +#include "prlog.h" #ifdef MOZ_OGG #include "OggWriter.h" @@ -12,50 +14,30 @@ #ifdef MOZ_OPUS #include "OpusTrackEncoder.h" #endif +#ifdef MOZ_WEBM_ENCODER +#include "VorbisTrackEncoder.h" +#include "VP8TrackEncoder.h" +#include "WebMWriter.h" +#endif +#ifdef MOZ_OMX_ENCODER +#include "OmxTrackEncoder.h" +#include "ISOMediaWriter.h" +#endif -#ifdef MOZ_WIDGET_GONK -#include -#define LOG(args...) __android_log_print(ANDROID_LOG_INFO, "MediaEncoder", ## args); +#ifdef LOG +#undef LOG +#endif + +#ifdef PR_LOGGING +PRLogModuleInfo* gMediaEncoderLog; +#define LOG(type, msg) PR_LOG(gMediaEncoderLog, type, msg) #else -#define LOG(args,...) +#define LOG(type, msg) #endif namespace mozilla { -#define TRACK_BUFFER_LEN 8192 - -namespace { - -template -static bool -TypeListContains(char const *const * aTypes, const String& aType) -{ - for (int32_t i = 0; aTypes[i]; ++i) { - if (aType.EqualsASCII(aTypes[i])) - return true; - } - return false; -} - -#ifdef MOZ_OGG -// The recommended mime-type for Ogg Opus files is audio/ogg. -// See http://wiki.xiph.org/OggOpus for more details. -static const char* const gOggTypes[2] = { - "audio/ogg", - nullptr -}; - -static bool -IsOggType(const nsAString& aType) -{ - if (!MediaDecoder::IsOggEnabled()) { - return false; - } - - return TypeListContains(gOggTypes, aType); -} -#endif -} //anonymous namespace +static nsIThread* sEncoderThread = nullptr; void MediaEncoder::NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, @@ -67,13 +49,15 @@ MediaEncoder::NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, { // Process the incoming raw track data from MediaStreamGraph, called on the // thread of MediaStreamGraph. - if (aQueuedMedia.GetType() == MediaSegment::AUDIO) { + if (mAudioEncoder && aQueuedMedia.GetType() == MediaSegment::AUDIO) { mAudioEncoder->NotifyQueuedTrackChanges(aGraph, aID, aTrackRate, aTrackOffset, aTrackEvents, aQueuedMedia); - } else { - // Type video is not supported for now. + } else if (mVideoEncoder && aQueuedMedia.GetType() == MediaSegment::VIDEO) { + mVideoEncoder->NotifyQueuedTrackChanges(aGraph, aID, aTrackRate, + aTrackOffset, aTrackEvents, + aQueuedMedia); } } @@ -81,49 +65,88 @@ void MediaEncoder::NotifyRemoved(MediaStreamGraph* aGraph) { // In case that MediaEncoder does not receive a TRACK_EVENT_ENDED event. - LOG("NotifyRemoved in [MediaEncoder]."); - mAudioEncoder->NotifyRemoved(aGraph); + LOG(PR_LOG_DEBUG, ("NotifyRemoved in [MediaEncoder].")); + if (mAudioEncoder) { + mAudioEncoder->NotifyRemoved(aGraph); + } + if (mVideoEncoder) { + mVideoEncoder->NotifyRemoved(aGraph); + } + } +bool +MediaEncoder::OnEncoderThread() +{ + return NS_GetCurrentThread() == sEncoderThread; +} /* static */ already_AddRefed -MediaEncoder::CreateEncoder(const nsAString& aMIMEType) +MediaEncoder::CreateEncoder(const nsAString& aMIMEType, uint8_t aTrackTypes) { +#ifdef PR_LOGGING + if (!gMediaEncoderLog) { + gMediaEncoderLog = PR_NewLogModule("MediaEncoder"); + } +#endif nsAutoPtr writer; nsAutoPtr audioEncoder; nsAutoPtr videoEncoder; nsRefPtr encoder; - - if (aMIMEType.IsEmpty()) { - // TODO: Should pick out a default container+codec base on the track - // coming from MediaStreamGraph. For now, just default to Ogg+Opus. - const_cast(aMIMEType) = NS_LITERAL_STRING("audio/ogg"); + nsString mimeType; + if (!aTrackTypes) { + LOG(PR_LOG_ERROR, ("NO TrackTypes!!!")); + return nullptr; } - - bool isAudioOnly = FindInReadable(NS_LITERAL_STRING("audio/"), aMIMEType); -#ifdef MOZ_OGG - if (IsOggType(aMIMEType)) { - writer = new OggWriter(); - if (!isAudioOnly) { - // Initialize the videoEncoder. +#ifdef MOZ_WEBM_ENCODER + else if (MediaDecoder::IsWebMEnabled() && + (aMIMEType.EqualsLiteral(VIDEO_WEBM) || + (aTrackTypes & ContainerWriter::HAS_VIDEO))) { + if (aTrackTypes & ContainerWriter::HAS_AUDIO) { + audioEncoder = new VorbisTrackEncoder(); + NS_ENSURE_TRUE(audioEncoder, nullptr); } -#ifdef MOZ_OPUS - audioEncoder = new OpusTrackEncoder(); -#endif - } -#endif - // If the given mime-type is video but fail to create the video encoder. - if (!isAudioOnly) { + videoEncoder = new VP8TrackEncoder(); + writer = new WebMWriter(aTrackTypes); + NS_ENSURE_TRUE(writer, nullptr); NS_ENSURE_TRUE(videoEncoder, nullptr); + mimeType = NS_LITERAL_STRING(VIDEO_WEBM); } - - // Return null if we fail to create the audio encoder. - NS_ENSURE_TRUE(audioEncoder, nullptr); - +#endif //MOZ_WEBM_ENCODER +#ifdef MOZ_OMX_ENCODER + else if (aMIMEType.EqualsLiteral(VIDEO_MP4) || + (aTrackTypes & ContainerWriter::HAS_VIDEO)) { + if (aTrackTypes & ContainerWriter::HAS_AUDIO) { + audioEncoder = new OmxAudioTrackEncoder(); + NS_ENSURE_TRUE(audioEncoder, nullptr); + } + videoEncoder = new OmxVideoTrackEncoder(); + writer = new ISOMediaWriter(aTrackTypes); + NS_ENSURE_TRUE(writer, nullptr); + NS_ENSURE_TRUE(videoEncoder, nullptr); + mimeType = NS_LITERAL_STRING(VIDEO_MP4); + } +#endif // MOZ_OMX_ENCODER +#ifdef MOZ_OGG + else if (MediaDecoder::IsOggEnabled() && MediaDecoder::IsOpusEnabled() && + (aMIMEType.EqualsLiteral(AUDIO_OGG) || + (aTrackTypes & ContainerWriter::HAS_AUDIO))) { + writer = new OggWriter(); + audioEncoder = new OpusTrackEncoder(); + NS_ENSURE_TRUE(writer, nullptr); + NS_ENSURE_TRUE(audioEncoder, nullptr); + mimeType = NS_LITERAL_STRING(AUDIO_OGG); + } +#endif // MOZ_OGG + else { + LOG(PR_LOG_ERROR, ("Can not find any encoder to record this media stream")); + return nullptr; + } + LOG(PR_LOG_DEBUG, ("Create encoder result:a[%d] v[%d] w[%d] mimeType = %s.", + audioEncoder != nullptr, videoEncoder != nullptr, + writer != nullptr, mimeType.get())); encoder = new MediaEncoder(writer.forget(), audioEncoder.forget(), - videoEncoder.forget(), aMIMEType); - - + videoEncoder.forget(), mimeType); return encoder.forget(); } @@ -156,75 +179,75 @@ MediaEncoder::GetEncodedData(nsTArray >* aOutputBufs, nsAString& aMIMEType) { MOZ_ASSERT(!NS_IsMainThread()); - + if (!sEncoderThread) { + sEncoderThread = NS_GetCurrentThread(); + } aMIMEType = mMIMEType; bool reloop = true; while (reloop) { switch (mState) { case ENCODE_METADDATA: { - nsRefPtr meta = mAudioEncoder->GetMetadata(); - if (meta == nullptr) { - LOG("ERROR! AudioEncoder get null Metadata!"); - mState = ENCODE_ERROR; + LOG(PR_LOG_DEBUG, ("ENCODE_METADDATA TimeStamp = %f", GetEncodeTimeStamp())); + nsresult rv = CopyMetadataToMuxer(mAudioEncoder.get()); + if (NS_FAILED(rv)) { + LOG(PR_LOG_ERROR, ("Error! Fail to Set Audio Metadata")); break; } - nsresult rv = mWriter->SetMetadata(meta); + rv = CopyMetadataToMuxer(mVideoEncoder.get()); if (NS_FAILED(rv)) { - LOG("ERROR! writer can't accept audio metadata!"); - mState = ENCODE_ERROR; - break; + LOG(PR_LOG_ERROR, ("Error! Fail to Set Video Metadata")); + break; } rv = mWriter->GetContainerData(aOutputBufs, ContainerWriter::GET_HEADER); if (NS_FAILED(rv)) { - LOG("ERROR! writer fail to generate header!"); + LOG(PR_LOG_ERROR,("Error! writer fail to generate header!")); mState = ENCODE_ERROR; break; } - + LOG(PR_LOG_DEBUG, ("Finish ENCODE_METADDATA TimeStamp = %f", GetEncodeTimeStamp())); mState = ENCODE_TRACK; break; } case ENCODE_TRACK: { + LOG(PR_LOG_DEBUG, ("ENCODE_TRACK TimeStamp = %f", GetEncodeTimeStamp())); EncodedFrameContainer encodedData; - nsresult rv = mAudioEncoder->GetEncodedTrack(encodedData); + nsresult rv = NS_OK; + rv = WriteEncodedDataToMuxer(mAudioEncoder.get()); if (NS_FAILED(rv)) { - // Encoding might be canceled. - LOG("ERROR! Fail to get encoded data from encoder."); - mState = ENCODE_ERROR; + LOG(PR_LOG_ERROR, ("Error! Fail to write audio encoder data to muxer")); break; } - rv = mWriter->WriteEncodedTrack(encodedData, - mAudioEncoder->IsEncodingComplete() ? - ContainerWriter::END_OF_STREAM : 0); + LOG(PR_LOG_DEBUG, ("Audio encoded TimeStamp = %f", GetEncodeTimeStamp())); + rv = WriteEncodedDataToMuxer(mVideoEncoder.get()); if (NS_FAILED(rv)) { - LOG("ERROR! Fail to write encoded track to the media container."); - mState = ENCODE_ERROR; + LOG(PR_LOG_ERROR, ("Fail to write video encoder data to muxer")); break; } - + LOG(PR_LOG_DEBUG, ("Video encoded TimeStamp = %f", GetEncodeTimeStamp())); + // In audio only or video only case, let unavailable track's flag to be true. + bool isAudioCompleted = (mAudioEncoder && mAudioEncoder->IsEncodingComplete()) || !mAudioEncoder; + bool isVideoCompleted = (mVideoEncoder && mVideoEncoder->IsEncodingComplete()) || !mVideoEncoder; rv = mWriter->GetContainerData(aOutputBufs, - mAudioEncoder->IsEncodingComplete() ? + isAudioCompleted && isVideoCompleted ? ContainerWriter::FLUSH_NEEDED : 0); if (NS_SUCCEEDED(rv)) { // Successfully get the copy of final container data from writer. reloop = false; } - - mState = (mAudioEncoder->IsEncodingComplete()) ? ENCODE_DONE : ENCODE_TRACK; + mState = (mWriter->IsWritingComplete()) ? ENCODE_DONE : ENCODE_TRACK; + LOG(PR_LOG_DEBUG, ("END ENCODE_TRACK TimeStamp = %f " + "mState = %d aComplete %d vComplete %d", + GetEncodeTimeStamp(), mState, isAudioCompleted, isVideoCompleted)); break; } case ENCODE_DONE: - LOG("MediaEncoder has been shutdown."); - mShutdown = true; - reloop = false; - break; case ENCODE_ERROR: - LOG("ERROR! MediaEncoder got error!"); + LOG(PR_LOG_DEBUG, ("MediaEncoder has been shutdown.")); mShutdown = true; reloop = false; break; @@ -234,4 +257,52 @@ MediaEncoder::GetEncodedData(nsTArray >* aOutputBufs, } } +nsresult +MediaEncoder::WriteEncodedDataToMuxer(TrackEncoder *aTrackEncoder) +{ + if (aTrackEncoder == nullptr) { + return NS_OK; + } + if (aTrackEncoder->IsEncodingComplete()) { + return NS_OK; + } + EncodedFrameContainer encodedVideoData; + nsresult rv = aTrackEncoder->GetEncodedTrack(encodedVideoData); + if (NS_FAILED(rv)) { + // Encoding might be canceled. + LOG(PR_LOG_ERROR, ("Error! Fail to get encoded data from video encoder.")); + mState = ENCODE_ERROR; + return rv; + } + rv = mWriter->WriteEncodedTrack(encodedVideoData, + aTrackEncoder->IsEncodingComplete() ? + ContainerWriter::END_OF_STREAM : 0); + if (NS_FAILED(rv)) { + LOG(PR_LOG_ERROR, ("Error! Fail to write encoded video track to the media container.")); + mState = ENCODE_ERROR; + } + return rv; +} + +nsresult +MediaEncoder::CopyMetadataToMuxer(TrackEncoder *aTrackEncoder) +{ + if (aTrackEncoder == nullptr) { + return NS_OK; + } + nsRefPtr meta = aTrackEncoder->GetMetadata(); + if (meta == nullptr) { + LOG(PR_LOG_ERROR, ("Error! metadata = null")); + mState = ENCODE_ERROR; + return NS_ERROR_ABORT; + } + + nsresult rv = mWriter->SetMetadata(meta); + if (NS_FAILED(rv)) { + LOG(PR_LOG_ERROR, ("Error! SetMetadata fail")); + mState = ENCODE_ERROR; + } + return rv; +} + } diff --git a/content/media/encoder/MediaEncoder.h b/content/media/encoder/MediaEncoder.h index 89e71556df2b..e82dc6e6d6fb 100644 --- a/content/media/encoder/MediaEncoder.h +++ b/content/media/encoder/MediaEncoder.h @@ -64,6 +64,7 @@ public : : mWriter(aWriter) , mAudioEncoder(aAudioEncoder) , mVideoEncoder(aVideoEncoder) + , mStartTime(TimeStamp::Now()) , mMIMEType(aMIMEType) , mState(MediaEncoder::ENCODE_METADDATA) , mShutdown(false) @@ -91,8 +92,12 @@ public : * to create the encoder. For now, default aMIMEType to "audio/ogg" and use * Ogg+Opus if it is empty. */ - static already_AddRefed CreateEncoder(const nsAString& aMIMEType); - + static already_AddRefed CreateEncoder(const nsAString& aMIMEType, + uint8_t aTrackTypes = ContainerWriter::HAS_AUDIO); + /** + * Check if run on Encoder thread + */ + static bool OnEncoderThread(); /** * Encodes the raw track data and returns the final container data. Assuming * it is called on a single worker thread. The buffer of container data is @@ -128,12 +133,24 @@ public : } private: + // Get encoded data from trackEncoder and write to muxer + nsresult WriteEncodedDataToMuxer(TrackEncoder *aTrackEncoder); + // Get metadata from trackEncoder and copy to muxer + nsresult CopyMetadataToMuxer(TrackEncoder* aTrackEncoder); nsAutoPtr mWriter; nsAutoPtr mAudioEncoder; nsAutoPtr mVideoEncoder; + TimeStamp mStartTime; nsString mMIMEType; int mState; bool mShutdown; + // Get duration from create encoder, for logging purpose + double GetEncodeTimeStamp() + { + TimeDuration decodeTime; + decodeTime = TimeStamp::Now() - mStartTime; + return decodeTime.ToMilliseconds(); + } }; } diff --git a/content/media/ogg/OggWriter.cpp b/content/media/ogg/OggWriter.cpp index f429238d0267..39917b0685fa 100644 --- a/content/media/ogg/OggWriter.cpp +++ b/content/media/ogg/OggWriter.cpp @@ -162,7 +162,9 @@ OggWriter::GetContainerData(nsTArray >* aOutputBufs, if (rc) { ProduceOggPage(aOutputBufs); } - + if (aFlags & ContainerWriter::FLUSH_NEEDED) { + mIsWritingComplete = true; + } return (rc > 0) ? NS_OK : NS_ERROR_FAILURE; } diff --git a/content/media/ogg/OggWriter.h b/content/media/ogg/OggWriter.h index e7426bb20447..c2658c1d32db 100644 --- a/content/media/ogg/OggWriter.h +++ b/content/media/ogg/OggWriter.h @@ -31,7 +31,6 @@ public: // Check metadata type integrity and reject unacceptable track encoder. nsresult SetMetadata(TrackMetadataBase* aMetadata) MOZ_OVERRIDE; - private: nsresult Init();