From d30ff7d8d12441a6e8e7f9c56022589f099a024a Mon Sep 17 00:00:00 2001 From: Ralph Giles Date: Thu, 19 Jun 2014 15:59:00 -0700 Subject: [PATCH] Bug 1019291 - Construct Annex B samples in the decoder. r=cpeace Move Annex B sample formatting from the demuxer to a static utility function. Return NAL unit syntax samples by default, since more platforms prefer to unpack the AVCC data themselves. Pass the raw AVCC payload to ffmpeg through its extradata field. It can handle either sample format, expecting one or the other depending on whether extradata is present. Pass input samples through a new WMFOutputSource::Input method. Save a reference to the VideoDecoderConfig in the WFMVideoOutputSource and use it to convert samples to Annex B before forwarding them to the MFTDecoder. --- .../media/fmp4/ffmpeg/FFmpegDataDecoder.cpp | 3 +++ content/media/fmp4/ffmpeg/FFmpegDataDecoder.h | 2 ++ .../media/fmp4/ffmpeg/FFmpegH264Decoder.cpp | 1 + .../media/fmp4/wmf/WMFAudioOutputSource.cpp | 9 +++++++ content/media/fmp4/wmf/WMFAudioOutputSource.h | 2 ++ content/media/fmp4/wmf/WMFDecoderModule.cpp | 3 ++- .../media/fmp4/wmf/WMFMediaDataDecoder.cpp | 6 ++--- content/media/fmp4/wmf/WMFMediaDataDecoder.h | 7 +++++ .../media/fmp4/wmf/WMFVideoOutputSource.cpp | 22 ++++++++++++--- content/media/fmp4/wmf/WMFVideoOutputSource.h | 7 ++++- media/libstagefright/binding/AnnexB.cpp | 27 +++++++++++++++---- media/libstagefright/binding/DecoderData.cpp | 2 +- .../binding/include/mp4_demuxer/AnnexB.h | 19 +++++++++---- .../binding/include/mp4_demuxer/DecoderData.h | 3 ++- .../binding/include/mp4_demuxer/mp4_demuxer.h | 4 +-- media/libstagefright/binding/mp4_demuxer.cpp | 11 +++----- .../media/libstagefright/MPEG4Extractor.cpp | 18 ++++++------- media/libstagefright/moz.build | 1 + 18 files changed, 107 insertions(+), 40 deletions(-) diff --git a/content/media/fmp4/ffmpeg/FFmpegDataDecoder.cpp b/content/media/fmp4/ffmpeg/FFmpegDataDecoder.cpp index 0068e83b8e29..cfed3ac59d49 100644 --- a/content/media/fmp4/ffmpeg/FFmpegDataDecoder.cpp +++ b/content/media/fmp4/ffmpeg/FFmpegDataDecoder.cpp @@ -87,6 +87,9 @@ FFmpegDataDecoder::Init() // FFmpeg will call back to this to negotiate a video pixel format. mCodecContext.get_format = ChoosePixelFormat; + mCodecContext.extradata = mExtraData.begin(); + mCodecContext.extradata_size = mExtraData.length(); + AVDictionary* opts = nullptr; if (avcodec_open2(&mCodecContext, codec, &opts) < 0) { NS_WARNING("Couldn't initialise ffmpeg decoder"); diff --git a/content/media/fmp4/ffmpeg/FFmpegDataDecoder.h b/content/media/fmp4/ffmpeg/FFmpegDataDecoder.h index 7816ab00e3be..1bebdb140858 100644 --- a/content/media/fmp4/ffmpeg/FFmpegDataDecoder.h +++ b/content/media/fmp4/ffmpeg/FFmpegDataDecoder.h @@ -10,6 +10,7 @@ #include "FFmpegDecoderModule.h" #include "FFmpegRuntimeLinker.h" #include "FFmpegCompat.h" +#include "mozilla/Vector.h" namespace mozilla { @@ -31,6 +32,7 @@ public: protected: MediaTaskQueue* mTaskQueue; AVCodecContext mCodecContext; + Vector mExtraData; private: static bool sFFmpegInitDone; diff --git a/content/media/fmp4/ffmpeg/FFmpegH264Decoder.cpp b/content/media/fmp4/ffmpeg/FFmpegH264Decoder.cpp index 68c633e5267c..25bb9bfc0dcb 100644 --- a/content/media/fmp4/ffmpeg/FFmpegH264Decoder.cpp +++ b/content/media/fmp4/ffmpeg/FFmpegH264Decoder.cpp @@ -33,6 +33,7 @@ FFmpegH264Decoder::FFmpegH264Decoder( , mImageContainer(aImageContainer) { MOZ_COUNT_CTOR(FFmpegH264Decoder); + mExtraData.append(aConfig.extra_data.begin(), aConfig.extra_data.length()); } nsresult diff --git a/content/media/fmp4/wmf/WMFAudioOutputSource.cpp b/content/media/fmp4/wmf/WMFAudioOutputSource.cpp index 575747279a7e..829176b7c777 100644 --- a/content/media/fmp4/wmf/WMFAudioOutputSource.cpp +++ b/content/media/fmp4/wmf/WMFAudioOutputSource.cpp @@ -5,6 +5,7 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "WMFAudioOutputSource.h" +#include "mp4_demuxer/DecoderData.h" #include "VideoUtils.h" #include "WMFUtils.h" #include "nsTArray.h" @@ -127,6 +128,14 @@ WMFAudioOutputSource::Init() return decoder.forget(); } +HRESULT +WMFAudioOutputSource::Input(mp4_demuxer::MP4Sample* aSample) +{ + const uint8_t* data = reinterpret_cast(aSample->data); + uint32_t length = aSample->size; + return mDecoder->Input(data, length, aSample->composition_timestamp); +} + HRESULT WMFAudioOutputSource::Output(int64_t aStreamOffset, nsAutoPtr& aOutData) diff --git a/content/media/fmp4/wmf/WMFAudioOutputSource.h b/content/media/fmp4/wmf/WMFAudioOutputSource.h index 73d326613a5b..e3298057f697 100644 --- a/content/media/fmp4/wmf/WMFAudioOutputSource.h +++ b/content/media/fmp4/wmf/WMFAudioOutputSource.h @@ -22,6 +22,8 @@ public: virtual TemporaryRef Init() MOZ_OVERRIDE; + virtual HRESULT Input(mp4_demuxer::MP4Sample* aSample) MOZ_OVERRIDE; + // Note WMF's AAC decoder sometimes output negatively timestamped samples, // presumably they're the preroll samples, and we strip them. We may return // a null aOutput in this case. diff --git a/content/media/fmp4/wmf/WMFDecoderModule.cpp b/content/media/fmp4/wmf/WMFDecoderModule.cpp index 174c336d5d65..6542a1ba4762 100644 --- a/content/media/fmp4/wmf/WMFDecoderModule.cpp +++ b/content/media/fmp4/wmf/WMFDecoderModule.cpp @@ -70,7 +70,8 @@ WMFDecoderModule::CreateH264Decoder(const mp4_demuxer::VideoDecoderConfig& aConf MediaTaskQueue* aVideoTaskQueue, MediaDataDecoderCallback* aCallback) { - return new WMFMediaDataDecoder(new WMFVideoOutputSource(aLayersBackend, + return new WMFMediaDataDecoder(new WMFVideoOutputSource(aConfig, + aLayersBackend, aImageContainer, sDXVAEnabled), aVideoTaskQueue, diff --git a/content/media/fmp4/wmf/WMFMediaDataDecoder.cpp b/content/media/fmp4/wmf/WMFMediaDataDecoder.cpp index 7ee0ee13172c..b2c7df625ec7 100644 --- a/content/media/fmp4/wmf/WMFMediaDataDecoder.cpp +++ b/content/media/fmp4/wmf/WMFMediaDataDecoder.cpp @@ -67,11 +67,9 @@ WMFMediaDataDecoder::Input(mp4_demuxer::MP4Sample* aSample) void WMFMediaDataDecoder::ProcessDecode(mp4_demuxer::MP4Sample* aSample) { - const uint8_t* data = reinterpret_cast(aSample->data); - uint32_t length = aSample->size; - HRESULT hr = mDecoder->Input(data, length, aSample->composition_timestamp); + HRESULT hr = mSource->Input(aSample); if (FAILED(hr)) { - NS_WARNING("WMFAudioDecoder failed to input data"); + NS_WARNING("WMFOutputSource rejected sample"); mCallback->Error(); return; } diff --git a/content/media/fmp4/wmf/WMFMediaDataDecoder.h b/content/media/fmp4/wmf/WMFMediaDataDecoder.h index e5a5019b2b24..28312411e3e1 100644 --- a/content/media/fmp4/wmf/WMFMediaDataDecoder.h +++ b/content/media/fmp4/wmf/WMFMediaDataDecoder.h @@ -13,6 +13,8 @@ #include "MFTDecoder.h" #include "mozilla/RefPtr.h" +class mp4_demuxer::MP4Sample; + namespace mozilla { // Encapsulates the initialization of the MFTDecoder appropriate for decoding @@ -26,6 +28,11 @@ public: // Returns nullptr on failure. virtual TemporaryRef Init() = 0; + // Submit a compressed sample for decoding. + // This should forward to the MFTDecoder after performing + // any required sample formatting. + virtual HRESULT Input(mp4_demuxer::MP4Sample* aSample) = 0; + // Produces decoded output, if possible. Blocks until output can be produced, // or until no more is able to be produced. // Returns S_OK on success, or MF_E_TRANSFORM_NEED_MORE_INPUT if there's not diff --git a/content/media/fmp4/wmf/WMFVideoOutputSource.cpp b/content/media/fmp4/wmf/WMFVideoOutputSource.cpp index c1fbd8d98651..c4a31b0cd68a 100644 --- a/content/media/fmp4/wmf/WMFVideoOutputSource.cpp +++ b/content/media/fmp4/wmf/WMFVideoOutputSource.cpp @@ -13,6 +13,8 @@ #include "nsThreadUtils.h" #include "Layers.h" #include "mozilla/layers/LayersTypes.h" +#include "mp4_demuxer/AnnexB.h" +#include "mp4_demuxer/DecoderData.h" #include "prlog.h" #include "gfx2DGlue.h" @@ -30,12 +32,15 @@ using mozilla::layers::LayersBackend; namespace mozilla { -WMFVideoOutputSource::WMFVideoOutputSource(mozilla::layers::LayersBackend aLayersBackend, - mozilla::layers::ImageContainer* aImageContainer, - bool aDXVAEnabled) +WMFVideoOutputSource::WMFVideoOutputSource( + const mp4_demuxer::VideoDecoderConfig& aConfig, + mozilla::layers::LayersBackend aLayersBackend, + mozilla::layers::ImageContainer* aImageContainer, + bool aDXVAEnabled) : mVideoStride(0) , mVideoWidth(0) , mVideoHeight(0) + , mConfig(aConfig) , mImageContainer(aImageContainer) , mDXVAEnabled(aDXVAEnabled) , mLayersBackend(aLayersBackend) @@ -138,6 +143,17 @@ WMFVideoOutputSource::Init() return decoder.forget(); } +HRESULT +WMFVideoOutputSource::Input(mp4_demuxer::MP4Sample* aSample) +{ + // We must prepare samples in AVC Annex B. + mp4_demuxer::AnnexB::ConvertSample(aSample, mConfig.annex_b); + // Forward sample data to the decoder. + const uint8_t* data = reinterpret_cast(aSample->data); + uint32_t length = aSample->size; + return mDecoder->Input(data, length, aSample->composition_timestamp); +} + HRESULT WMFVideoOutputSource::ConfigureVideoFrameGeometry() { diff --git a/content/media/fmp4/wmf/WMFVideoOutputSource.h b/content/media/fmp4/wmf/WMFVideoOutputSource.h index 07fee720c6ac..c74af3f013d9 100644 --- a/content/media/fmp4/wmf/WMFVideoOutputSource.h +++ b/content/media/fmp4/wmf/WMFVideoOutputSource.h @@ -20,13 +20,16 @@ class DXVA2Manager; class WMFVideoOutputSource : public WMFOutputSource { public: - WMFVideoOutputSource(mozilla::layers::LayersBackend aLayersBackend, + WMFVideoOutputSource(const mp4_demuxer::VideoDecoderConfig& aConfig, + mozilla::layers::LayersBackend aLayersBackend, mozilla::layers::ImageContainer* aImageContainer, bool aDXVAEnabled); ~WMFVideoOutputSource(); virtual TemporaryRef Init() MOZ_OVERRIDE; + virtual HRESULT Input(mp4_demuxer::MP4Sample* aSample) MOZ_OVERRIDE; + virtual HRESULT Output(int64_t aStreamOffset, nsAutoPtr& aOutput) MOZ_OVERRIDE; @@ -51,6 +54,8 @@ private: uint32_t mVideoHeight; nsIntRect mPictureRegion; + const mp4_demuxer::VideoDecoderConfig& mConfig; + RefPtr mDecoder; RefPtr mImageContainer; nsAutoPtr mDXVA2Manager; diff --git a/media/libstagefright/binding/AnnexB.cpp b/media/libstagefright/binding/AnnexB.cpp index d676fa6b33e3..80cc5c696d5f 100644 --- a/media/libstagefright/binding/AnnexB.cpp +++ b/media/libstagefright/binding/AnnexB.cpp @@ -2,6 +2,7 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#include "mozilla/ArrayUtils.h" #include "mp4_demuxer/AnnexB.h" #include "mp4_demuxer/ByteReader.h" #include "mp4_demuxer/DecoderData.h" @@ -13,6 +14,21 @@ namespace mp4_demuxer static const uint8_t kAnnexBDelimiter[] = { 0, 0, 0, 1 }; +void +AnnexB::ConvertSample(MP4Sample* aSample, + const mozilla::Vector& annexB) +{ + MOZ_ASSERT(aSample); + MOZ_ASSERT(aSample->data); + MOZ_ASSERT(aSample->size >= ArrayLength(kAnnexBDelimiter)); + // Overwrite the NAL length with the Annex B separator. + memcpy(aSample->data, kAnnexBDelimiter, ArrayLength(kAnnexBDelimiter)); + // Prepend the Annex B header with SPS and PPS tables to keyframes. + if (aSample->is_sync_point) { + aSample->Prepend(annexB.begin(), annexB.length()); + } +} + Vector AnnexB::ConvertExtraDataToAnnexB(mozilla::Vector& aExtraData) { @@ -36,9 +52,9 @@ AnnexB::ConvertExtraDataToAnnexB(mozilla::Vector& aExtraData) ByteReader reader(aExtraData); const uint8_t* ptr = reader.Read(5); if (ptr && ptr[0] == 1) { - // Append SPS then PSP - ConvertSpsOrPsp(reader, reader.ReadU8() & 31, &annexB); - ConvertSpsOrPsp(reader, reader.ReadU8(), &annexB); + // Append SPS then PPS + ConvertSPSOrPPS(reader, reader.ReadU8() & 31, &annexB); + ConvertSPSOrPPS(reader, reader.ReadU8(), &annexB); MOZ_ASSERT(!reader.Remaining()); } @@ -47,7 +63,7 @@ AnnexB::ConvertExtraDataToAnnexB(mozilla::Vector& aExtraData) } void -AnnexB::ConvertSpsOrPsp(ByteReader& aReader, uint8_t aCount, +AnnexB::ConvertSPSOrPPS(ByteReader& aReader, uint8_t aCount, Vector* aAnnexB) { for (int i = 0; i < aCount; i++) { @@ -62,4 +78,5 @@ AnnexB::ConvertSpsOrPsp(ByteReader& aReader, uint8_t aCount, aAnnexB->append(ptr, length); } } -} + +} // namespace mp4_demuxer diff --git a/media/libstagefright/binding/DecoderData.cpp b/media/libstagefright/binding/DecoderData.cpp index e0845a2adbf5..c86bf08668a8 100644 --- a/media/libstagefright/binding/DecoderData.cpp +++ b/media/libstagefright/binding/DecoderData.cpp @@ -83,7 +83,7 @@ VideoDecoderConfig::Update(sp& aMetaData, const char* aMimeType) uint32_t type; if (aMetaData->findData(kKeyAVCC, &type, &data, &size)) { - mozilla::Vector extra_data; + extra_data.clear(); extra_data.append(reinterpret_cast(data), size); annex_b = AnnexB::ConvertExtraDataToAnnexB(extra_data); } diff --git a/media/libstagefright/binding/include/mp4_demuxer/AnnexB.h b/media/libstagefright/binding/include/mp4_demuxer/AnnexB.h index ed631daa448e..f4be4df5f49b 100644 --- a/media/libstagefright/binding/include/mp4_demuxer/AnnexB.h +++ b/media/libstagefright/binding/include/mp4_demuxer/AnnexB.h @@ -2,24 +2,33 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -#ifndef ANNEX_B_H_ -#define ANNEX_B_H_ +#ifndef MP4_DEMUXER_ANNEX_B_H_ +#define MP4_DEMUXER_ANNEX_B_H_ #include "mozilla/Vector.h" namespace mp4_demuxer { class ByteReader; +class MP4Sample; + class AnnexB { public: + // Convert a sample from NAL unit syntax to Annex B. + static void ConvertSample(MP4Sample* aSample, + const mozilla::Vector& annexB); + + // Parse an AVCC box and construct the Annex B sample header. static mozilla::Vector ConvertExtraDataToAnnexB( mozilla::Vector& aExtraData); private: - static void ConvertSpsOrPsp(ByteReader& aReader, uint8_t aCount, + // AVCC box parser helper. + static void ConvertSPSOrPPS(ByteReader& aReader, uint8_t aCount, mozilla::Vector* aAnnexB); }; -} -#endif +} // namespace mp4_demuxer + +#endif // MP4_DEMUXER_ANNEX_B_H_ diff --git a/media/libstagefright/binding/include/mp4_demuxer/DecoderData.h b/media/libstagefright/binding/include/mp4_demuxer/DecoderData.h index 1c39f81da52a..81d38c4fcee3 100644 --- a/media/libstagefright/binding/include/mp4_demuxer/DecoderData.h +++ b/media/libstagefright/binding/include/mp4_demuxer/DecoderData.h @@ -67,7 +67,8 @@ public: int32_t display_width; int32_t display_height; - mozilla::Vector annex_b; + mozilla::Vector extra_data; // Unparsed AVCDecoderConfig payload. + mozilla::Vector annex_b; // Parsed version for sample prepend. void Update(stagefright::sp& aMetaData, const char* aMimeType); bool IsValid(); diff --git a/media/libstagefright/binding/include/mp4_demuxer/mp4_demuxer.h b/media/libstagefright/binding/include/mp4_demuxer/mp4_demuxer.h index 8b9953de33b4..a60af43cbe16 100644 --- a/media/libstagefright/binding/include/mp4_demuxer/mp4_demuxer.h +++ b/media/libstagefright/binding/include/mp4_demuxer/mp4_demuxer.h @@ -44,8 +44,8 @@ public: void SeekAudio(Microseconds aTime); void SeekVideo(Microseconds aTime); - // DemuxAudioSample and DemuxVideoSample functions return nullptr on end of - // stream or error. + // DemuxAudioSample and DemuxVideoSample functions + // return nullptr on end of stream or error. MP4Sample* DemuxAudioSample(); MP4Sample* DemuxVideoSample(); diff --git a/media/libstagefright/binding/mp4_demuxer.cpp b/media/libstagefright/binding/mp4_demuxer.cpp index f55fc7fd3a60..51c2cef53dfd 100644 --- a/media/libstagefright/binding/mp4_demuxer.cpp +++ b/media/libstagefright/binding/mp4_demuxer.cpp @@ -65,7 +65,8 @@ private: nsAutoPtr mSource; }; -MP4Demuxer::MP4Demuxer(Stream* source) : mPrivate(new StageFrightPrivate()) +MP4Demuxer::MP4Demuxer(Stream* source) + : mPrivate(new StageFrightPrivate()) { mPrivate->mExtractor = new MPEG4Extractor(new DataSourceAdapter(source)); } @@ -180,11 +181,7 @@ MP4Demuxer::DemuxVideoSample() sample->Update(); - if (sample->is_sync_point) { - sample->Prepend(mVideoConfig.annex_b.begin(), - mVideoConfig.annex_b.length()); - } - return sample.forget(); } -} + +} // namespace mp4_demuxer diff --git a/media/libstagefright/frameworks/av/media/libstagefright/MPEG4Extractor.cpp b/media/libstagefright/frameworks/av/media/libstagefright/MPEG4Extractor.cpp index cc600f698d9c..b53584bb498f 100644 --- a/media/libstagefright/frameworks/av/media/libstagefright/MPEG4Extractor.cpp +++ b/media/libstagefright/frameworks/av/media/libstagefright/MPEG4Extractor.cpp @@ -3301,11 +3301,10 @@ status_t MPEG4Source::read( } CHECK(dstOffset + 4 <= mBuffer->size()); - - dstData[dstOffset++] = 0; - dstData[dstOffset++] = 0; - dstData[dstOffset++] = 0; - dstData[dstOffset++] = 1; + dstData[dstOffset++] = (uint8_t) (nalLength >> 24); + dstData[dstOffset++] = (uint8_t) (nalLength >> 16); + dstData[dstOffset++] = (uint8_t) (nalLength >> 8); + dstData[dstOffset++] = (uint8_t) nalLength; memcpy(&dstData[dstOffset], &mSrcBuffer[srcOffset], nalLength); srcOffset += nalLength; dstOffset += nalLength; @@ -3589,11 +3588,10 @@ status_t MPEG4Source::fragmentedRead( } CHECK(dstOffset + 4 <= mBuffer->size()); - - dstData[dstOffset++] = 0; - dstData[dstOffset++] = 0; - dstData[dstOffset++] = 0; - dstData[dstOffset++] = 1; + dstData[dstOffset++] = (uint8_t) (nalLength >> 24); + dstData[dstOffset++] = (uint8_t) (nalLength >> 16); + dstData[dstOffset++] = (uint8_t) (nalLength >> 8); + dstData[dstOffset++] = (uint8_t) nalLength; memcpy(&dstData[dstOffset], &mSrcBuffer[srcOffset], nalLength); srcOffset += nalLength; dstOffset += nalLength; diff --git a/media/libstagefright/moz.build b/media/libstagefright/moz.build index ba2295f4c239..f991d7f283b4 100644 --- a/media/libstagefright/moz.build +++ b/media/libstagefright/moz.build @@ -46,6 +46,7 @@ if CONFIG['OS_TARGET'] != 'Android': ] EXPORTS.mp4_demuxer += [ + 'binding/include/mp4_demuxer/AnnexB.h', 'binding/include/mp4_demuxer/DecoderData.h', 'binding/include/mp4_demuxer/mp4_demuxer.h', ]