Bug 1174981: P1. Ensure frames are returned in pts order. r=gerald

We move management of the data to the TrackBuffersManager as it contains
precise information on how the samples are being moved.
This commit is contained in:
Jean-Yves Avenard 2015-06-16 12:14:29 +10:00
Родитель 4c70b3dc19
Коммит 03db89712a
4 изменённых файлов: 266 добавлений и 115 удалений

Просмотреть файл

@ -17,6 +17,11 @@ typedef TrackInfo::TrackType TrackType;
using media::TimeUnit;
using media::TimeIntervals;
// Gap allowed between frames. Due to inaccuracies in determining buffer end
// frames (Bug 1065207). This value is based on the end of frame
// default value used in Blink, kDefaultBufferDurationInMs.
#define EOS_FUZZ_US 125000
MediaSourceDemuxer::MediaSourceDemuxer()
: mTaskQueue(new MediaTaskQueue(GetMediaThreadPool(MediaThreadType::PLAYBACK),
/* aSupportsTailDispatch = */ true))
@ -225,7 +230,6 @@ MediaSourceTrackDemuxer::MediaSourceTrackDemuxer(MediaSourceDemuxer* aParent,
: mParent(aParent)
, mManager(aManager)
, mType(aType)
, mNextSampleIndex(0)
, mMonitor("MediaSourceTrackDemuxer")
{
}
@ -259,11 +263,11 @@ MediaSourceTrackDemuxer::Reset()
nsRefPtr<MediaSourceTrackDemuxer> self = this;
nsCOMPtr<nsIRunnable> task =
NS_NewRunnableFunction([self] () {
self->mNextSampleTime = TimeUnit();
self->mNextSampleIndex = 0;
self->mManager->Seek(self->mType, TimeUnit());
{
MonitorAutoLock mon(self->mMonitor);
self->mNextRandomAccessPoint = self->GetNextRandomAccessPoint();
self->mNextRandomAccessPoint =
self->mManager->GetNextRandomAccessPoint(self->mType);
}
});
mParent->GetTaskQueue()->Dispatch(task.forget());
@ -310,47 +314,39 @@ MediaSourceTrackDemuxer::BreakCycles()
nsRefPtr<MediaSourceTrackDemuxer::SeekPromise>
MediaSourceTrackDemuxer::DoSeek(media::TimeUnit aTime)
{
if (aTime.ToMicroseconds() && !mManager->Buffered(mType).Contains(aTime)) {
if (aTime.ToMicroseconds() && !mBufferedRanges.Contains(aTime)) {
// We don't have the data to seek to.
return SeekPromise::CreateAndReject(DemuxerFailureReason::WAITING_FOR_DATA,
__func__);
}
const TrackBuffersManager::TrackBuffer& track =
mManager->GetTrackBuffer(mType);
TimeUnit lastKeyFrameTime;
uint32_t lastKeyFrameIndex = 0;
for (uint32_t i = 0; i < track.Length(); i++) {
const nsRefPtr<MediaRawData>& sample = track[i];
if (sample->mKeyframe) {
lastKeyFrameTime = TimeUnit::FromMicroseconds(sample->mTime);
lastKeyFrameIndex = i;
}
if (sample->mTime >= aTime.ToMicroseconds()) {
break;
}
}
mNextSampleIndex = lastKeyFrameIndex;
mNextSampleTime = lastKeyFrameTime;
TimeUnit seekTime = mManager->Seek(mType, aTime);
{
MonitorAutoLock mon(mMonitor);
mNextRandomAccessPoint = GetNextRandomAccessPoint();
mNextRandomAccessPoint = mManager->GetNextRandomAccessPoint(mType);
}
return SeekPromise::CreateAndResolve(mNextSampleTime, __func__);
return SeekPromise::CreateAndResolve(seekTime, __func__);
}
nsRefPtr<MediaSourceTrackDemuxer::SamplesPromise>
MediaSourceTrackDemuxer::DoGetSamples(int32_t aNumSamples)
{
DemuxerFailureReason failure;
nsRefPtr<MediaRawData> sample = GetSample(failure);
bool error;
nsRefPtr<MediaRawData> sample = mManager->GetSample(mType,
TimeUnit::FromMicroseconds(EOS_FUZZ_US),
error);
if (!sample) {
return SamplesPromise::CreateAndReject(failure, __func__);
if (error) {
return SamplesPromise::CreateAndReject(DemuxerFailureReason::DEMUXER_ERROR, __func__);
}
return SamplesPromise::CreateAndReject(
mManager->IsEnded() ? DemuxerFailureReason::END_OF_STREAM :
DemuxerFailureReason::WAITING_FOR_DATA, __func__);
}
nsRefPtr<SamplesHolder> samples = new SamplesHolder;
samples->mSamples.AppendElement(sample);
if (mNextRandomAccessPoint.ToMicroseconds() <= sample->mTime) {
MonitorAutoLock mon(mMonitor);
mNextRandomAccessPoint = GetNextRandomAccessPoint();
mNextRandomAccessPoint = mManager->GetNextRandomAccessPoint(mType);
}
return SamplesPromise::CreateAndResolve(samples, __func__);
}
@ -358,21 +354,9 @@ MediaSourceTrackDemuxer::DoGetSamples(int32_t aNumSamples)
nsRefPtr<MediaSourceTrackDemuxer::SkipAccessPointPromise>
MediaSourceTrackDemuxer::DoSkipToNextRandomAccessPoint(media::TimeUnit aTimeThreadshold)
{
bool found = false;
int32_t parsed = 0;
const TrackBuffersManager::TrackBuffer& track =
mManager->GetTrackBuffer(mType);
for (uint32_t i = mNextSampleIndex; i < track.Length(); i++) {
const nsRefPtr<MediaRawData>& sample = track[i];
if (sample->mKeyframe &&
sample->mTime >= aTimeThreadshold.ToMicroseconds()) {
mNextSampleTime = TimeUnit::FromMicroseconds(sample->GetEndTime());
found = true;
break;
}
parsed++;
}
bool found;
uint32_t parsed =
mManager->SkipToNextRandomAccessPoint(mType, aTimeThreadshold, found);
if (found) {
return SkipAccessPointPromise::CreateAndResolve(parsed, __func__);
}
@ -382,70 +366,6 @@ MediaSourceTrackDemuxer::DoSkipToNextRandomAccessPoint(media::TimeUnit aTimeThre
return SkipAccessPointPromise::CreateAndReject(holder, __func__);
}
already_AddRefed<MediaRawData>
MediaSourceTrackDemuxer::GetSample(DemuxerFailureReason& aFailure)
{
const TrackBuffersManager::TrackBuffer& track =
mManager->GetTrackBuffer(mType);
const TimeIntervals& ranges = mManager->Buffered(mType);
if (mNextSampleTime >= ranges.GetEnd()) {
if (mManager->IsEnded()) {
aFailure = DemuxerFailureReason::END_OF_STREAM;
} else {
aFailure = DemuxerFailureReason::WAITING_FOR_DATA;
}
return nullptr;
}
if (mNextSampleTime == TimeUnit()) {
// First demux, get first sample time.
mNextSampleTime = ranges.GetStart();
}
if (!ranges.ContainsWithStrictEnd(mNextSampleTime)) {
aFailure = DemuxerFailureReason::WAITING_FOR_DATA;
return nullptr;
}
if (mNextSampleIndex) {
const nsRefPtr<MediaRawData>& sample = track[mNextSampleIndex];
nsRefPtr<MediaRawData> p = sample->Clone();
if (!p) {
aFailure = DemuxerFailureReason::DEMUXER_ERROR;
return nullptr;
}
mNextSampleTime = TimeUnit::FromMicroseconds(sample->GetEndTime());
mNextSampleIndex++;
return p.forget();
}
for (uint32_t i = 0; i < track.Length(); i++) {
const nsRefPtr<MediaRawData>& sample = track[i];
if (sample->mTime >= mNextSampleTime.ToMicroseconds()) {
nsRefPtr<MediaRawData> p = sample->Clone();
mNextSampleTime = TimeUnit::FromMicroseconds(sample->GetEndTime());
mNextSampleIndex = i+1;
if (!p) {
// OOM
break;
}
return p.forget();
}
}
aFailure = DemuxerFailureReason::DEMUXER_ERROR;
return nullptr;
}
TimeUnit
MediaSourceTrackDemuxer::GetNextRandomAccessPoint()
{
const TrackBuffersManager::TrackBuffer& track = mManager->GetTrackBuffer(mType);
for (uint32_t i = mNextSampleIndex; i < track.Length(); i++) {
const nsRefPtr<MediaRawData>& sample = track[i];
if (sample->mKeyframe) {
return TimeUnit::FromMicroseconds(sample->mTime);
}
}
return media::TimeUnit::FromInfinity();
}
void
MediaSourceTrackDemuxer::NotifyTimeRangesChanged()
{
@ -453,7 +373,8 @@ MediaSourceTrackDemuxer::NotifyTimeRangesChanged()
return;
}
MOZ_ASSERT(mParent->OnTaskQueue());
mNextSampleIndex = 0;
mBufferedRanges = mManager->Buffered(mType);
mBufferedRanges.SetFuzz(TimeUnit::FromMicroseconds(EOS_FUZZ_US));
}
} // namespace mozilla

Просмотреть файл

@ -127,8 +127,7 @@ private:
nsRefPtr<MediaSourceDemuxer> mParent;
nsRefPtr<TrackBuffersManager> mManager;
TrackInfo::TrackType mType;
uint32_t mNextSampleIndex;
media::TimeUnit mNextSampleTime;
media::TimeIntervals mBufferedRanges;
// Monitor protecting members below accessed from multiple threads.
Monitor mMonitor;
media::TimeUnit mNextRandomAccessPoint;

Просмотреть файл

@ -66,6 +66,10 @@ TrackBuffersManager::TrackBuffersManager(dom::SourceBuffer* aParent, MediaSource
GetTaskQueue()->Dispatch(task.forget());
}
TrackBuffersManager::~TrackBuffersManager()
{
}
bool
TrackBuffersManager::AppendData(MediaByteBuffer* aData,
TimeUnit aTimestampOffset)
@ -485,6 +489,15 @@ TrackBuffersManager::CodedFrameRemoval(TimeInterval aInterval)
track->mNextInsertionIndex.ref() > i) {
track->mNextInsertionIndex.ref()--;
}
if (track->mNextGetSampleIndex.isSome()) {
if (track->mNextGetSampleIndex.ref() == i) {
MSE_DEBUG("Next sample to be played got evicted");
track->mNextGetSampleIndex.reset();
} else if (track->mNextGetSampleIndex.ref() > i) {
track->mNextGetSampleIndex.ref()--;
}
}
} else {
i++;
}
@ -505,6 +518,7 @@ TrackBuffersManager::CodedFrameRemoval(TimeInterval aInterval)
track->mSizeBuffer -= sizeof(*sample) + sample->mSize;
}
data.RemoveElementsAt(start, end - start);
removeCurrentCodedFrameGroup |=
track->mNextInsertionIndex.isSome() &&
track->mNextInsertionIndex.ref() >= start &&
@ -515,6 +529,16 @@ TrackBuffersManager::CodedFrameRemoval(TimeInterval aInterval)
track->mNextInsertionIndex.ref() -= end - start;
}
if (track->mNextGetSampleIndex.isSome()) {
if (track->mNextGetSampleIndex.ref() >= start &&
track->mNextGetSampleIndex.ref() < end) {
MSE_DEBUG("Next sample to be played got evicted");
track->mNextGetSampleIndex.reset();
} else if (track->mNextGetSampleIndex.ref() >= end) {
track->mNextGetSampleIndex.ref() -= end - start;
}
}
MSE_DEBUG("Removing undecodable frames from:%u (frames:%d) ([%f, %f))",
start, end - start,
removedInterval.mStart.ToSeconds(), removedInterval.mEnd.ToSeconds());
@ -1196,7 +1220,6 @@ bool
TrackBuffersManager::ProcessFrame(MediaRawData* aSample,
TrackData& aTrackData)
{
TrackData* tracks[] = { &mVideoTracks, &mAudioTracks };
TimeUnit presentationTimestamp;
TimeUnit decodeTimestamp;
@ -1262,7 +1285,7 @@ TrackBuffersManager::ProcessFrame(MediaRawData* aSample,
// Set group start timestamp equal to the group end timestamp.
mGroupStartTimestamp = Some(mGroupEndTimestamp);
}
for (auto& track : tracks) {
for (auto& track : GetTracksList()) {
// 2. Unset the last decode timestamp on all track buffers.
// 3. Unset the last frame duration on all track buffers.
// 4. Unset the highest end timestamp on all track buffers.
@ -1347,6 +1370,15 @@ TrackBuffersManager::ProcessFrame(MediaRawData* aSample,
TimeUnit::FromMicroseconds(sample->mTime).ToSeconds(),
TimeUnit::FromMicroseconds(sample->GetEndTime()).ToSeconds());
data.RemoveElementAt(i);
if (trackBuffer.mNextGetSampleIndex.isSome()) {
if (trackBuffer.mNextGetSampleIndex.ref() == i) {
MSE_DEBUG("Next sample to be played got evicted");
trackBuffer.mNextGetSampleIndex.reset();
} else if (trackBuffer.mNextGetSampleIndex.ref() > i) {
trackBuffer.mNextGetSampleIndex.ref()--;
}
}
} else {
i++;
}
@ -1373,6 +1405,16 @@ TrackBuffersManager::ProcessFrame(MediaRawData* aSample,
start, end - start,
removedInterval.mStart.ToSeconds(), removedInterval.mEnd.ToSeconds());
if (trackBuffer.mNextGetSampleIndex.isSome()) {
if (trackBuffer.mNextGetSampleIndex.ref() >= start &&
trackBuffer.mNextGetSampleIndex.ref() < end) {
MSE_DEBUG("Next sample to be played got evicted");
trackBuffer.mNextGetSampleIndex.reset();
} else if (trackBuffer.mNextGetSampleIndex.ref() >= end) {
trackBuffer.mNextGetSampleIndex.ref() -= end - start;
}
}
// Update our buffered range to exclude the range just removed.
trackBuffer.mBufferedRanges -= removedInterval;
MOZ_ASSERT(trackBuffer.mNextInsertionIndex.isNothing() ||
@ -1552,10 +1594,6 @@ TrackBuffersManager::RestartGroupStartTimestamp()
mGroupStartTimestamp = Some(mGroupEndTimestamp);
}
TrackBuffersManager::~TrackBuffersManager()
{
}
MediaInfo
TrackBuffersManager::GetMetadata()
{
@ -1577,5 +1615,182 @@ TrackBuffersManager::GetTrackBuffer(TrackInfo::TrackType aTrack)
return GetTracksData(aTrack).mBuffers.LastElement();
}
TimeUnit
TrackBuffersManager::Seek(TrackInfo::TrackType aTrack,
const TimeUnit& aTime)
{
MOZ_ASSERT(OnTaskQueue());
auto& trackBuffer = GetTracksData(aTrack);
const TrackBuffersManager::TrackBuffer& track = GetTrackBuffer(aTrack);
TimeUnit lastKeyFrameTime;
TimeUnit lastKeyFrameTimecode;
uint32_t lastKeyFrameIndex = 0;
for (uint32_t i = 0; i < track.Length(); i++) {
const nsRefPtr<MediaRawData>& sample = track[i];
TimeUnit sampleTime = TimeUnit::FromMicroseconds(sample->mTime);
if (sampleTime > aTime) {
break;
}
if (sample->mKeyframe) {
lastKeyFrameTimecode = TimeUnit::FromMicroseconds(sample->mTimecode);
lastKeyFrameTime = sampleTime;
lastKeyFrameIndex = i;
}
if (sampleTime == aTime) {
break;
}
}
trackBuffer.mNextGetSampleIndex = Some(lastKeyFrameIndex);
trackBuffer.mNextSampleTimecode = lastKeyFrameTimecode;
trackBuffer.mNextSampleTime = lastKeyFrameTime;
return lastKeyFrameTime;
}
uint32_t
TrackBuffersManager::SkipToNextRandomAccessPoint(TrackInfo::TrackType aTrack,
const TimeUnit& aTimeThreadshold,
bool& aFound)
{
MOZ_ASSERT(OnTaskQueue());
uint32_t parsed = 0;
auto& trackData = GetTracksData(aTrack);
const TrackBuffer& track = GetTrackBuffer(aTrack);
uint32_t nextSampleIndex = trackData.mNextGetSampleIndex.valueOr(0);
for (uint32_t i = nextSampleIndex; i < track.Length(); i++) {
const nsRefPtr<MediaRawData>& sample = track[i];
if (sample->mKeyframe &&
sample->mTime >= aTimeThreadshold.ToMicroseconds()) {
trackData.mNextSampleTimecode =
TimeUnit::FromMicroseconds(sample->mTimecode);
trackData.mNextSampleTime =
TimeUnit::FromMicroseconds(sample->mTime);
trackData.mNextGetSampleIndex = Some(i);
aFound = true;
break;
}
parsed++;
}
return parsed;
}
already_AddRefed<MediaRawData>
TrackBuffersManager::GetSample(TrackInfo::TrackType aTrack,
const TimeUnit& aFuzz,
bool& aError)
{
MOZ_ASSERT(OnTaskQueue());
auto& trackData = GetTracksData(aTrack);
const TrackBuffer& track = GetTrackBuffer(aTrack);
aError = false;
if (!track.Length() ||
(trackData.mNextGetSampleIndex.isSome() &&
trackData.mNextGetSampleIndex.ref() >= track.Length())) {
return nullptr;
}
if (trackData.mNextGetSampleIndex.isNothing() &&
trackData.mNextSampleTimecode == TimeUnit()) {
// First demux, get first sample.
trackData.mNextGetSampleIndex = Some(0u);
}
if (trackData.mNextGetSampleIndex.isSome()) {
const nsRefPtr<MediaRawData>& sample =
track[trackData.mNextGetSampleIndex.ref()];
if (trackData.mNextGetSampleIndex.ref() &&
sample->mTimecode > (trackData.mNextSampleTimecode + aFuzz).ToMicroseconds()) {
// Gap is too big. End of Stream or Waiting for Data.
return nullptr;
}
nsRefPtr<MediaRawData> p = sample->Clone();
if (!p) {
aError = true;
return nullptr;
}
trackData.mNextGetSampleIndex.ref()++;
// Estimate decode timestamp of the next sample.
trackData.mNextSampleTimecode =
TimeUnit::FromMicroseconds(sample->mTimecode + sample->mDuration);
trackData.mNextSampleTime =
TimeUnit::FromMicroseconds(sample->GetEndTime());
return p.forget();
}
// Our previous index has been overwritten, attempt to find the new one.
for (uint32_t i = 0; i < track.Length(); i++) {
const nsRefPtr<MediaRawData>& sample = track[i];
TimeInterval sampleInterval{
TimeUnit::FromMicroseconds(sample->mTimecode),
TimeUnit::FromMicroseconds(sample->mTimecode + sample->mDuration),
aFuzz};
if (sampleInterval.ContainsWithStrictEnd(trackData.mNextSampleTimecode)) {
nsRefPtr<MediaRawData> p = sample->Clone();
if (!p) {
// OOM
aError = true;
return nullptr;
}
trackData.mNextGetSampleIndex = Some(i+1);
trackData.mNextSampleTimecode = sampleInterval.mEnd;
trackData.mNextSampleTime =
TimeUnit::FromMicroseconds(sample->GetEndTime());
return p.forget();
}
}
// We couldn't find our sample by decode timestamp. Attempt to find it using
// presentation timestamp. There will likely be small jerkiness.
for (uint32_t i = 0; i < track.Length(); i++) {
const nsRefPtr<MediaRawData>& sample = track[i];
TimeInterval sampleInterval{
TimeUnit::FromMicroseconds(sample->mTime),
TimeUnit::FromMicroseconds(sample->GetEndTime()),
aFuzz};
if (sampleInterval.ContainsWithStrictEnd(trackData.mNextSampleTimecode)) {
nsRefPtr<MediaRawData> p = sample->Clone();
if (!p) {
// OOM
aError = true;
return nullptr;
}
trackData.mNextGetSampleIndex = Some(i+1);
// Estimate decode timestamp of the next sample.
trackData.mNextSampleTimecode = sampleInterval.mEnd;
trackData.mNextSampleTime =
TimeUnit::FromMicroseconds(sample->GetEndTime());
return p.forget();
}
}
MSE_DEBUG("Couldn't find sample (pts:%lld dts:%lld)",
trackData.mNextSampleTime.ToMicroseconds(),
trackData.mNextSampleTimecode.ToMicroseconds());
return nullptr;
}
TimeUnit
TrackBuffersManager::GetNextRandomAccessPoint(TrackInfo::TrackType aTrack)
{
auto& trackData = GetTracksData(aTrack);
MOZ_ASSERT(trackData.mNextGetSampleIndex.isSome());
const TrackBuffersManager::TrackBuffer& track = GetTrackBuffer(aTrack);
uint32_t i = trackData.mNextGetSampleIndex.ref();
for (; i < track.Length(); i++) {
const nsRefPtr<MediaRawData>& sample = track[i];
if (sample->mKeyframe) {
return TimeUnit::FromMicroseconds(sample->mTime);
}
}
return media::TimeUnit::FromInfinity();
}
}
#undef MSE_DEBUG

Просмотреть файл

@ -8,6 +8,7 @@
#define MOZILLA_TRACKBUFFERSMANAGER_H_
#include "SourceBufferContentManager.h"
#include "MediaDataDemuxer.h"
#include "MediaSourceDecoder.h"
#include "mozilla/Atomics.h"
#include "mozilla/Maybe.h"
@ -79,6 +80,14 @@ public:
{
return mEnded;
}
TimeUnit Seek(TrackInfo::TrackType aTrack, const TimeUnit& aTime);
uint32_t SkipToNextRandomAccessPoint(TrackInfo::TrackType aTrack,
const TimeUnit& aTimeThreadshold,
bool& aFound);
already_AddRefed<MediaRawData> GetSample(TrackInfo::TrackType aTrack,
const TimeUnit& aFuzz,
bool& aError);
TimeUnit GetNextRandomAccessPoint(TrackInfo::TrackType aTrack);
#if defined(DEBUG)
void Dump(const char* aPath) override;
@ -226,6 +235,13 @@ private:
// TrackInfo of the last metadata parsed (updated with each init segment.
nsRefPtr<SharedTrackInfo> mLastInfo;
// If set, position of the next sample to be retrieved by GetSample().
Maybe<uint32_t> mNextGetSampleIndex;
// Approximation of the next sample's decode timestamp.
TimeUnit mNextSampleTimecode;
// Approximation of the next sample's presentation timestamp.
TimeUnit mNextSampleTime;
void ResetAppendState()
{
mLastDecodeTimestamp.reset();