Bug 1267887 - Support Opus in mp4 with the rust demuxer. r=kinetik

Update C++ caller code for for mp4parse 0.4.0. Now feeds data through
a read callback in mp4parse_io.

Hook up the GetTrackInfo method to the rust demuxer results.

Prefer rust demuxer only if there's an Opus track.

Fill in audio and video track metadata. Pass audio codec_specific_config
to the decoder.

With this change sample.mp4 plays.

MozReview-Commit-ID: F8xwWPZZBfZ
This commit is contained in:
Ralph Giles 2016-04-01 15:44:00 -07:00
Родитель c1c627c22a
Коммит 72ecfe6999
8 изменённых файлов: 361 добавлений и 74 удалений

Просмотреть файл

@ -93,7 +93,8 @@ MP4Decoder::CanHandleMediaType(const nsACString& aMIMETypeExcludingCodecs,
// the web, as opposed to what we use internally (i.e. what our demuxers
// etc output).
const bool isMP4Audio = aMIMETypeExcludingCodecs.EqualsASCII("audio/mp4") ||
aMIMETypeExcludingCodecs.EqualsASCII("audio/x-m4a");
aMIMETypeExcludingCodecs.EqualsASCII("audio/x-m4a") ||
aMIMETypeExcludingCodecs.EqualsASCII("audio/opus");
const bool isMP4Video =
// On B2G, treat 3GPP as MP4 when Gonk PDM is available.
#ifdef MOZ_GONK_MEDIACODEC

Просмотреть файл

@ -5,6 +5,7 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "AgnosticDecoderModule.h"
#include "mozilla/Logging.h"
#include "OpusDecoder.h"
#include "VorbisDecoder.h"
#include "VPXDecoder.h"
@ -16,10 +17,14 @@ bool
AgnosticDecoderModule::SupportsMimeType(const nsACString& aMimeType,
DecoderDoctorDiagnostics* aDiagnostics) const
{
return VPXDecoder::IsVPX(aMimeType) ||
bool supports =
VPXDecoder::IsVPX(aMimeType) ||
OpusDataDecoder::IsOpus(aMimeType) ||
VorbisDataDecoder::IsVorbis(aMimeType) ||
WaveDataDecoder::IsWave(aMimeType);
MOZ_LOG(sPDMLog, LogLevel::Debug, ("Agnostic decoder %s requested type",
supports ? "supports" : "rejects"));
return supports;
}
already_AddRefed<MediaDataDecoder>

Просмотреть файл

@ -348,7 +348,9 @@ bool
OpusDataDecoder::IsOpus(const nsACString& aMimeType)
{
return aMimeType.EqualsLiteral("audio/webm; codecs=opus") ||
aMimeType.EqualsLiteral("audio/ogg; codecs=opus");
aMimeType.EqualsLiteral("audio/ogg; codecs=opus") ||
aMimeType.EqualsLiteral("audio/mp4; codecs=opus") ||
aMimeType.EqualsLiteral("audio/opus");
}
} // namespace mozilla

Просмотреть файл

@ -13,6 +13,10 @@
#include "mozilla/ArrayUtils.h"
#include "include/ESDS.h"
#ifdef MOZ_RUST_MP4PARSE
#include "mp4parse.h"
#endif
using namespace stagefright;
namespace mp4_demuxer
@ -182,6 +186,26 @@ MP4VideoInfo::Update(const MetaData* aMetaData, const char* aMimeType)
}
#ifdef MOZ_RUST_MP4PARSE
void
MP4VideoInfo::Update(const mp4parse_track_info* track,
const mp4parse_track_video_info* video)
{
if (track->codec == MP4PARSE_CODEC_AVC) {
mMimeType = MEDIA_MIMETYPE_VIDEO_AVC;
} else if (track->codec == MP4PARSE_CODEC_VP9) {
mMimeType = NS_LITERAL_CSTRING("video/vp9");
}
mTrackId = track->track_id;
mDuration = track->duration;
mMediaTime = track->media_time;
mDisplay.width = video->display_width;
mDisplay.height = video->display_height;
mImage.width = video->image_width;
mImage.height = video->image_height;
}
#endif
bool
MP4VideoInfo::IsValid() const
{

Просмотреть файл

@ -7,10 +7,17 @@
#include "media/stagefright/MediaDefs.h"
#include "media/stagefright/MediaSource.h"
#include "media/stagefright/MetaData.h"
#include "mozilla/Assertions.h"
#include "mozilla/CheckedInt.h"
#include "mozilla/EndianUtils.h"
#include "mozilla/Logging.h"
#include "mozilla/RefPtr.h"
#include "mozilla/Telemetry.h"
#include "mozilla/UniquePtr.h"
#include "VideoUtils.h"
#include "mp4_demuxer/MoofParser.h"
#include "mp4_demuxer/MP4Metadata.h"
#include "mp4_demuxer/Stream.h"
#include <limits>
#include <stdint.h>
@ -19,7 +26,7 @@
#ifdef MOZ_RUST_MP4PARSE
#include "mp4parse.h"
struct FreeMP4ParseState { void operator()(mp4parse_state* aPtr) { mp4parse_free(aPtr); } };
struct FreeMP4Parser { void operator()(mp4parse_parser* aPtr) { mp4parse_free(aPtr); } };
#endif
using namespace stagefright;
@ -95,6 +102,26 @@ private:
};
#ifdef MOZ_RUST_MP4PARSE
// Wrap an mp4_demuxer::Stream to remember the read offset.
class RustStreamAdaptor {
public:
explicit RustStreamAdaptor(Stream* aSource)
: mSource(aSource)
, mOffset(0)
{
}
~RustStreamAdaptor() {}
bool Read(uint8_t* buffer, uintptr_t size, size_t* bytes_read);
private:
Stream* mSource;
CheckedInt<size_t> mOffset;
};
class MP4MetadataRust
{
public:
@ -115,7 +142,8 @@ public:
private:
CryptoFile mCrypto;
RefPtr<Stream> mSource;
mozilla::UniquePtr<mp4parse_state, FreeMP4ParseState> mRustState;
RustStreamAdaptor mRustSource;
mozilla::UniquePtr<mp4parse_parser, FreeMP4Parser> mRustParser;
};
#endif
@ -123,6 +151,7 @@ MP4Metadata::MP4Metadata(Stream* aSource)
: mStagefright(MakeUnique<MP4MetadataStagefright>(aSource))
#ifdef MOZ_RUST_MP4PARSE
, mRust(MakeUnique<MP4MetadataRust>(aSource))
, mPreferRust(false)
, mReportedTelemetry(false)
#endif
{
@ -186,16 +215,58 @@ MP4Metadata::GetNumberTracks(mozilla::TrackInfo::TrackType aType) const
mReportedTelemetry = true;
}
#endif
if (mPreferRust || ShouldPreferRust()) {
MOZ_LOG(sLog, LogLevel::Info, ("Preferring rust demuxer"));
mPreferRust = true;
return numTracksRust;
}
#endif // MOZ_RUST_MP4PARSE
return numTracks;
}
#ifdef MOZ_RUST_MP4PARSE
bool MP4Metadata::ShouldPreferRust() const {
if (!mRust) {
return false;
}
// See if there's an Opus track.
uint32_t numTracks = mRust->GetNumberTracks(TrackInfo::kAudioTrack);
for (auto i = 0; i < numTracks; i++) {
auto info = mRust->GetTrackInfo(TrackInfo::kAudioTrack, i);
if (!info) {
return false;
}
if (info->mMimeType.EqualsASCII("audio/opus")) {
return true;
}
}
// Otherwise, fall back.
return false;
}
#endif // MOZ_RUST_MP4PARSE
mozilla::UniquePtr<mozilla::TrackInfo>
MP4Metadata::GetTrackInfo(mozilla::TrackInfo::TrackType aType,
size_t aTrackNumber) const
{
return mStagefright->GetTrackInfo(aType, aTrackNumber);
mozilla::UniquePtr<mozilla::TrackInfo> info =
mStagefright->GetTrackInfo(aType, aTrackNumber);
#ifdef MOZ_RUST_MP4PARSE
if (!mRust || !mPreferRust) {
return info;
}
mozilla::UniquePtr<mozilla::TrackInfo> infoRust =
mRust->GetTrackInfo(aType, aTrackNumber);
MOZ_ASSERT(infoRust);
return infoRust;
#endif
return info;
}
bool
@ -371,6 +442,7 @@ MP4MetadataStagefright::CheckTrack(const char* aMimeType,
if (e && e->IsValid()) {
return e;
}
return nullptr;
}
@ -455,46 +527,55 @@ MP4MetadataStagefright::Metadata(Stream* aSource)
}
#ifdef MOZ_RUST_MP4PARSE
static int32_t
read_source(RefPtr<Stream> aSource, std::vector<uint8_t>& aBuffer)
bool
RustStreamAdaptor::Read(uint8_t* buffer, uintptr_t size, size_t* bytes_read)
{
static LazyLogModule sLog("MP4Metadata");
int64_t length;
if (!aSource->Length(&length) || length <= 0) {
MOZ_LOG(sLog, LogLevel::Warning, ("Couldn't get source length"));
return MP4PARSE_ERROR_IO;
if (!mOffset.isValid()) {
static LazyLogModule sLog("MP4Metadata");
MOZ_LOG(sLog, LogLevel::Error, ("Overflow in source stream offset"));
return false;
}
MOZ_LOG(sLog, LogLevel::Debug,
("Source length %d bytes\n", (long long int)length));
length = std::min<int64_t>(length, 1024 * 1024); // Don't read the entire file.
aBuffer.resize(length);
bool rv = mSource->ReadAt(mOffset.value(), buffer, size, bytes_read);
if (rv) {
mOffset += *bytes_read;
}
return rv;
}
// Wrapper to allow rust to call our read adaptor.
static intptr_t
read_source(uint8_t* buffer, uintptr_t size, void* userdata)
{
MOZ_ASSERT(buffer);
MOZ_ASSERT(userdata);
auto source = reinterpret_cast<RustStreamAdaptor*>(userdata);
size_t bytes_read = 0;
bool rv = aSource->ReadAt(0, aBuffer.data(), aBuffer.size(), &bytes_read);
if (!rv || bytes_read != size_t(length)) {
MOZ_LOG(sLog, LogLevel::Warning, ("Error copying mp4 data"));
return MP4PARSE_ERROR_IO;
bool rv = source->Read(buffer, size, &bytes_read);
if (!rv) {
static LazyLogModule sLog("MP4Metadata");
MOZ_LOG(sLog, LogLevel::Warning, ("Error reading source data"));
return -1;
}
return MP4PARSE_OK;
return bytes_read;
}
MP4MetadataRust::MP4MetadataRust(Stream* aSource)
: mSource(aSource)
, mRustState(mp4parse_new())
, mRustSource(aSource)
{
static LazyLogModule sLog("MP4Metadata");
mp4parse_io io = { read_source, &mRustSource };
mRustParser.reset(mp4parse_new(&io));
MOZ_ASSERT(mRustParser);
std::vector<uint8_t> buffer;
int32_t rv = read_source(mSource, buffer);
if (rv == MP4PARSE_OK) {
rv = mp4parse_read(mRustState.get(), buffer.data(), buffer.size());
}
static LazyLogModule sLog("MP4Metadata");
mp4parse_error rv = mp4parse_read(mRustParser.get());
MOZ_LOG(sLog, LogLevel::Debug, ("rust parser returned %d\n", rv));
Telemetry::Accumulate(Telemetry::MEDIA_RUST_MP4PARSE_SUCCESS,
rv == MP4PARSE_OK);
if (rv != MP4PARSE_OK) {
MOZ_ASSERT(rv > 0);
Telemetry::Accumulate(Telemetry::MEDIA_RUST_MP4PARSE_ERROR_CODE,
rv);
Telemetry::Accumulate(Telemetry::MEDIA_RUST_MP4PARSE_ERROR_CODE, rv);
}
}
@ -507,24 +588,30 @@ MP4MetadataRust::GetNumberTracks(mozilla::TrackInfo::TrackType aType) const
{
static LazyLogModule sLog("MP4Metadata");
uint32_t tracks = mp4parse_get_track_count(mRustState.get());
uint32_t tracks;
mp4parse_error rv = mp4parse_get_track_count(mRustParser.get(), &tracks);
if (rv != MP4PARSE_OK) {
MOZ_LOG(sLog, LogLevel::Warning,
("rust parser error %d counting tracks", rv));
return 0;
}
MOZ_LOG(sLog, LogLevel::Info, ("rust parser found %u tracks", tracks));
uint32_t total = 0;
for (uint32_t i = 0; i < tracks; ++i) {
mp4parse_track_info track_info;
int32_t rv = mp4parse_get_track_info(mRustState.get(), i, &track_info);
int32_t rv = mp4parse_get_track_info(mRustParser.get(), i, &track_info);
if (rv != MP4PARSE_OK) {
continue;
}
switch (aType) {
case mozilla::TrackInfo::kAudioTrack:
if (track_info.track_type == MP4PARSE_TRACK_TYPE_AAC) {
if (track_info.track_type == MP4PARSE_TRACK_TYPE_AUDIO) {
total += 1;
}
break;
case mozilla::TrackInfo::kVideoTrack:
if (track_info.track_type == MP4PARSE_TRACK_TYPE_H264) {
if (track_info.track_type == MP4PARSE_TRACK_TYPE_VIDEO) {
total += 1;
}
break;
@ -540,7 +627,90 @@ mozilla::UniquePtr<mozilla::TrackInfo>
MP4MetadataRust::GetTrackInfo(mozilla::TrackInfo::TrackType aType,
size_t aTrackNumber) const
{
MOZ_ASSERT(false, "Not yet implemented");
static LazyLogModule sLog("MP4Metadata");
mp4parse_track_info info;
auto rv = mp4parse_get_track_info(mRustParser.get(), aTrackNumber, &info);
if (rv != MP4PARSE_OK) {
MOZ_LOG(sLog, LogLevel::Warning, ("mp4parse_get_track_info returned %d", rv));
return nullptr;
}
#ifdef DEBUG
const char* codec_string = "unrecognized";
switch (info.codec) {
case MP4PARSE_CODEC_UNKNOWN: codec_string = "unknown"; break;
case MP4PARSE_CODEC_AAC: codec_string = "aac"; break;
case MP4PARSE_CODEC_OPUS: codec_string = "opus"; break;
case MP4PARSE_CODEC_AVC: codec_string = "h.264"; break;
case MP4PARSE_CODEC_VP9: codec_string = "vp9"; break;
}
MOZ_LOG(sLog, LogLevel::Debug, ("track codec %s (%u)\n",
codec_string, info.codec));
#endif
// This specialization interface is crazy.
UniquePtr<mozilla::TrackInfo> e;
switch (aType) {
case TrackInfo::TrackType::kAudioTrack: {
mp4parse_track_audio_info audio;
auto rv = mp4parse_get_track_audio_info(mRustParser.get(), aTrackNumber, &audio);
if (rv != MP4PARSE_OK) {
MOZ_LOG(sLog, LogLevel::Warning, ("mp4parse_get_track_audio_info returned error %d", rv));
return nullptr;
}
auto track = mozilla::MakeUnique<mozilla::AudioInfo>();
if (info.codec == MP4PARSE_CODEC_OPUS) {
track->mMimeType = NS_LITERAL_CSTRING("audio/opus");
// The Opus decoder expects the container's codec delay or
// pre-skip value, in microseconds, as a 64-bit int at the
// start of the codec-specific config blob.
MOZ_ASSERT(audio.codec_specific_config.data);
MOZ_ASSERT(audio.codec_specific_config.length >= 12);
uint16_t preskip =
LittleEndian::readUint16(audio.codec_specific_config.data + 10);
MOZ_LOG(sLog, LogLevel::Debug,
("Copying opus pre-skip value of %d as CodecDelay.",(int)preskip));
uint8_t codecDelay[sizeof(uint64_t)];
BigEndian::writeUint64(codecDelay,
mozilla::FramesToUsecs(preskip, 48000).value());
track->mCodecSpecificConfig->AppendElements(codecDelay, sizeof(uint64_t));
} else if (info.codec == MP4PARSE_CODEC_AAC) {
track->mMimeType = MEDIA_MIMETYPE_AUDIO_AAC;
}
track->mCodecSpecificConfig->AppendElements(
audio.codec_specific_config.data,
audio.codec_specific_config.length);
track->mRate = audio.sample_rate;
track->mChannels = audio.channels;
track->mBitDepth = audio.bit_depth;
track->mDuration = info.duration;
track->mMediaTime = info.media_time;
e = Move(track);
}
break;
case TrackInfo::TrackType::kVideoTrack: {
mp4parse_track_video_info video;
auto rv = mp4parse_get_track_video_info(mRustParser.get(), aTrackNumber, &video);
if (rv != MP4PARSE_OK) {
MOZ_LOG(sLog, LogLevel::Warning, ("mp4parse_get_track_audio_info returned error %d", rv));
return nullptr;
}
auto track = mozilla::MakeUnique<MP4VideoInfo>();
track->Update(&info, &video);
e = Move(track);
}
break;
default:
MOZ_LOG(sLog, LogLevel::Warning, ("unhandled track type %d", aType));
return nullptr;
break;
}
if (e && e->IsValid()) {
return e;
}
MOZ_LOG(sLog, LogLevel::Debug, ("TrackInfo didn't validate"));
return nullptr;
}

Просмотреть файл

@ -19,6 +19,14 @@ namespace stagefright
class MetaData;
}
#ifdef MOZ_RUST_MP4PARSE
extern "C" {
typedef struct mp4parse_track_info mp4parse_track_info;
typedef struct mp4parse_track_audio_info mp4parse_track_audio_info;
typedef struct mp4parse_track_video_info mp4parse_track_video_info;
}
#endif
namespace mp4_demuxer
{
@ -72,6 +80,11 @@ public:
void Update(const stagefright::MetaData* aMetaData,
const char* aMimeType);
#ifdef MOZ_RUST_MP4PARSE
void Update(const mp4parse_track_info* track,
const mp4parse_track_video_info* video);
#endif
virtual bool IsValid() const override;
};

Просмотреть файл

@ -39,7 +39,9 @@ private:
UniquePtr<MP4MetadataStagefright> mStagefright;
#ifdef MOZ_RUST_MP4PARSE
UniquePtr<MP4MetadataRust> mRust;
mutable bool mPreferRust;
mutable bool mReportedTelemetry;
bool ShouldPreferRust() const;
#endif
};

Просмотреть файл

@ -4,46 +4,112 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "gtest/gtest.h"
#include "mp4_demuxer/MP4Metadata.h"
#include "mp4parse.h"
#include <stdint.h>
#include <stdio.h>
#include <string.h>
#include <vector>
using namespace mp4_demuxer;
using namespace mozilla;
static intptr_t
error_reader(uint8_t* buffer, uintptr_t size, void* userdata)
{
return -1;
}
struct read_vector {
explicit read_vector(FILE* file, size_t length);
explicit read_vector(size_t length);
size_t location;
std::vector<uint8_t> buffer;
};
read_vector::read_vector(FILE* file, size_t length)
: location(0)
{
buffer.resize(length);
size_t read = fread(buffer.data(), sizeof(decltype(buffer)::value_type),
buffer.size(), file);
buffer.resize(read);
}
read_vector::read_vector(size_t length)
: location(0)
{
buffer.resize(length, 0);
}
static intptr_t
vector_reader(uint8_t* buffer, uintptr_t size, void* userdata)
{
if (!buffer || !userdata) {
return -1;
}
auto source = reinterpret_cast<read_vector*>(userdata);
if (source->location > source->buffer.size()) {
return -1;
}
uintptr_t available = source->buffer.size() - source->location;
uintptr_t length = std::min(available, size);
memcpy(buffer, source->buffer.data() + source->location, length);
source->location += length;
return length;
}
TEST(rust, MP4MetadataEmpty)
{
int32_t rv;
mp4parse_error rv;
mp4parse_io io;
mp4parse_state* context = mp4parse_new();
// Shouldn't be able to read with no context.
rv = mp4parse_read(nullptr);
EXPECT_EQ(rv, MP4PARSE_ERROR_BADARG);
// Shouldn't be able to wrap an mp4parse_io with null members.
io = { nullptr, nullptr };
mp4parse_parser* context = mp4parse_new(&io);
EXPECT_EQ(context, nullptr);
io = { nullptr, &io };
context = mp4parse_new(&io);
EXPECT_EQ(context, nullptr);
// FIXME: this should probably be accepted.
io = { error_reader, nullptr };
context = mp4parse_new(&io);
EXPECT_EQ(context, nullptr);
// Read method errors should propagate.
io = { error_reader, &io };
context = mp4parse_new(&io);
ASSERT_NE(context, nullptr);
rv = mp4parse_read(context);
EXPECT_EQ(rv, MP4PARSE_ERROR_IO);
mp4parse_free(context);
rv = mp4parse_read(nullptr, nullptr, 0);
EXPECT_EQ(rv, MP4PARSE_ERROR_BADARG);
rv = mp4parse_read(context, nullptr, 0);
EXPECT_EQ(rv, MP4PARSE_ERROR_BADARG);
// Short buffers should fail.
read_vector buf(0);
io = { vector_reader, &buf };
context = mp4parse_new(&io);
ASSERT_NE(context, nullptr);
rv = mp4parse_read(context);
EXPECT_EQ(rv, MP4PARSE_ERROR_INVALID);
mp4parse_free(context);
size_t len = 4097;
rv = mp4parse_read(nullptr, nullptr, len);
EXPECT_EQ(rv, MP4PARSE_ERROR_BADARG);
rv = mp4parse_read(context, nullptr, len);
EXPECT_EQ(rv, MP4PARSE_ERROR_BADARG);
std::vector<uint8_t> buf;
rv = mp4parse_read(nullptr, &buf.front(), buf.size());
EXPECT_EQ(rv, MP4PARSE_ERROR_BADARG);
rv = mp4parse_read(context, &buf.front(), buf.size());
EXPECT_EQ(rv, MP4PARSE_ERROR_BADARG);
buf.reserve(len);
rv = mp4parse_read(nullptr, &buf.front(), buf.size());
EXPECT_EQ(rv, MP4PARSE_ERROR_BADARG);
rv = mp4parse_read(context, &buf.front(), buf.size());
EXPECT_EQ(rv, MP4PARSE_ERROR_BADARG);
buf.buffer.reserve(4097);
context = mp4parse_new(&io);
ASSERT_NE(context, nullptr);
rv = mp4parse_read(context);
EXPECT_EQ(rv, MP4PARSE_ERROR_INVALID);
mp4parse_free(context);
// Empty buffers should fail.
buf.buffer.resize(4097, 0);
context = mp4parse_new(&io);
rv = mp4parse_read(context);
EXPECT_EQ(rv, MP4PARSE_ERROR_UNSUPPORTED);
mp4parse_free(context);
}
@ -51,21 +117,25 @@ TEST(rust, MP4Metadata)
{
FILE* f = fopen("street.mp4", "rb");
ASSERT_TRUE(f != nullptr);
size_t len = 4096;
std::vector<uint8_t> buf(len);
size_t read = fread(&buf.front(), sizeof(decltype(buf)::value_type), buf.size(), f);
buf.resize(read);
// Read just the moov header to work around the parser
// treating mid-box eof as an error.
//read_vector reader = read_vector(f, 1061);
struct stat s;
ASSERT_EQ(0, fstat(fileno(f), &s));
read_vector reader = read_vector(f, s.st_size);
fclose(f);
mp4parse_state* context = mp4parse_new();
ASSERT_NE(context, nullptr);
mp4parse_io io = { vector_reader, &reader };
mp4parse_parser* context = mp4parse_new(&io);
ASSERT_NE(nullptr, context);
int32_t rv = mp4parse_read(context, &buf.front(), buf.size());
EXPECT_EQ(rv, MP4PARSE_OK);
mp4parse_error rv = mp4parse_read(context);
EXPECT_EQ(MP4PARSE_OK, rv);
uint32_t tracks = mp4parse_get_track_count(context);
EXPECT_EQ(tracks, 2U);
uint32_t tracks = 0;
rv = mp4parse_get_track_count(context, &tracks);
EXPECT_EQ(MP4PARSE_OK, rv);
EXPECT_EQ(2U, tracks);
mp4parse_free(context);
}