Bug 1201363 - MediaStreamVideoSink for MediaRecorder case. r=jesup

Add MediaStreamVideoRecorderSink into MediaEncorder. In this patch, I still keep use duration to pass to TrackEncoders. Don't want to make this bug too big and out of control. We can file a new bug to change TrackEncoders use TimeStamp only.

MozReview-Commit-ID: KGftzulZynj

--HG--
extra : amend_source : 90fd7ddab4ecda4b95cd77c705cfb29692782de9
This commit is contained in:
Chia-hung Tai 2016-06-15 16:48:44 +01:00
Родитель 3d269ad415
Коммит 7d3f0cf614
11 изменённых файлов: 197 добавлений и 95 удалений

Просмотреть файл

@ -295,8 +295,10 @@ class MediaRecorder::Session: public nsIObserver,
class TracksAvailableCallback : public OnTracksAvailableCallback class TracksAvailableCallback : public OnTracksAvailableCallback
{ {
public: public:
explicit TracksAvailableCallback(Session *aSession) explicit TracksAvailableCallback(Session *aSession, TrackRate aTrackRate)
: mSession(aSession) {} : mSession(aSession)
, mTrackRate(aTrackRate) {}
virtual void NotifyTracksAvailable(DOMMediaStream* aStream) virtual void NotifyTracksAvailable(DOMMediaStream* aStream)
{ {
if (mSession->mStopIssued) { if (mSession->mStopIssued) {
@ -347,10 +349,11 @@ class MediaRecorder::Session: public nsIObserver,
} }
LOG(LogLevel::Debug, ("Session.NotifyTracksAvailable track type = (%d)", trackTypes)); LOG(LogLevel::Debug, ("Session.NotifyTracksAvailable track type = (%d)", trackTypes));
mSession->InitEncoder(trackTypes); mSession->InitEncoder(trackTypes, mTrackRate);
} }
private: private:
RefPtr<Session> mSession; RefPtr<Session> mSession;
TrackRate mTrackRate;
}; };
// Main thread task. // Main thread task.
// To delete RecordingSession object. // To delete RecordingSession object.
@ -412,6 +415,7 @@ public:
, mIsStartEventFired(false) , mIsStartEventFired(false)
, mIsRegisterProfiler(false) , mIsRegisterProfiler(false)
, mNeedSessionEndTask(true) , mNeedSessionEndTask(true)
, mSelectedVideoTrackID(TRACK_NONE)
{ {
MOZ_ASSERT(NS_IsMainThread()); MOZ_ASSERT(NS_IsMainThread());
MOZ_COUNT_CTOR(MediaRecorder::Session); MOZ_COUNT_CTOR(MediaRecorder::Session);
@ -467,6 +471,7 @@ public:
// Create a Track Union Stream // Create a Track Union Stream
MediaStreamGraph* gm = mRecorder->GetSourceMediaStream()->Graph(); MediaStreamGraph* gm = mRecorder->GetSourceMediaStream()->Graph();
TrackRate trackRate = gm->GraphRate();
mTrackUnionStream = gm->CreateTrackUnionStream(); mTrackUnionStream = gm->CreateTrackUnionStream();
MOZ_ASSERT(mTrackUnionStream, "CreateTrackUnionStream failed"); MOZ_ASSERT(mTrackUnionStream, "CreateTrackUnionStream failed");
@ -477,7 +482,7 @@ public:
// Get the available tracks from the DOMMediaStream. // Get the available tracks from the DOMMediaStream.
// The callback will report back tracks that we have to connect to // The callback will report back tracks that we have to connect to
// mTrackUnionStream and listen to principal changes on. // mTrackUnionStream and listen to principal changes on.
TracksAvailableCallback* tracksAvailableCallback = new TracksAvailableCallback(this); TracksAvailableCallback* tracksAvailableCallback = new TracksAvailableCallback(this, trackRate);
domStream->OnTracksAvailable(tracksAvailableCallback); domStream->OnTracksAvailable(tracksAvailableCallback);
} else { } else {
// Check that we may access the audio node's content. // Check that we may access the audio node's content.
@ -493,7 +498,7 @@ public:
MOZ_ASSERT(mInputPorts[mInputPorts.Length()-1]); MOZ_ASSERT(mInputPorts[mInputPorts.Length()-1]);
// Web Audio node has only audio. // Web Audio node has only audio.
InitEncoder(ContainerWriter::CREATE_AUDIO_TRACK); InitEncoder(ContainerWriter::CREATE_AUDIO_TRACK, trackRate);
} }
} }
@ -734,7 +739,7 @@ private:
return perm == nsIPermissionManager::ALLOW_ACTION; return perm == nsIPermissionManager::ALLOW_ACTION;
} }
void InitEncoder(uint8_t aTrackTypes) void InitEncoder(uint8_t aTrackTypes, TrackRate aTrackRate)
{ {
LOG(LogLevel::Debug, ("Session.InitEncoder %p", this)); LOG(LogLevel::Debug, ("Session.InitEncoder %p", this));
MOZ_ASSERT(NS_IsMainThread()); MOZ_ASSERT(NS_IsMainThread());
@ -752,19 +757,19 @@ private:
mRecorder->GetAudioBitrate(), mRecorder->GetAudioBitrate(),
mRecorder->GetVideoBitrate(), mRecorder->GetVideoBitrate(),
mRecorder->GetBitrate(), mRecorder->GetBitrate(),
aTrackTypes); aTrackTypes, aTrackRate);
} else if (mRecorder->mMimeType.EqualsLiteral(AUDIO_3GPP2) && CheckPermission("audio-capture:3gpp2")) { } else if (mRecorder->mMimeType.EqualsLiteral(AUDIO_3GPP2) && CheckPermission("audio-capture:3gpp2")) {
mEncoder = MediaEncoder::CreateEncoder(NS_LITERAL_STRING(AUDIO_3GPP2), mEncoder = MediaEncoder::CreateEncoder(NS_LITERAL_STRING(AUDIO_3GPP2),
mRecorder->GetAudioBitrate(), mRecorder->GetAudioBitrate(),
mRecorder->GetVideoBitrate(), mRecorder->GetVideoBitrate(),
mRecorder->GetBitrate(), mRecorder->GetBitrate(),
aTrackTypes); aTrackTypes, aTrackRate);
} else { } else {
mEncoder = MediaEncoder::CreateEncoder(NS_LITERAL_STRING(""), mEncoder = MediaEncoder::CreateEncoder(NS_LITERAL_STRING(""),
mRecorder->GetAudioBitrate(), mRecorder->GetAudioBitrate(),
mRecorder->GetVideoBitrate(), mRecorder->GetVideoBitrate(),
mRecorder->GetBitrate(), mRecorder->GetBitrate(),
aTrackTypes); aTrackTypes, aTrackRate);
} }
if (!mEncoder) { if (!mEncoder) {
@ -781,13 +786,25 @@ private:
DoSessionEndTask(NS_OK); DoSessionEndTask(NS_OK);
return; return;
} }
mTrackUnionStream->AddListener(mEncoder); mTrackUnionStream->AddListener(mEncoder.get());
// Try to use direct listeners if possible
nsTArray<RefPtr<mozilla::dom::VideoStreamTrack>> videoTracks;
DOMMediaStream* domStream = mRecorder->Stream(); DOMMediaStream* domStream = mRecorder->Stream();
if (domStream) {
domStream->GetVideoTracks(videoTracks);
if (!videoTracks.IsEmpty()) {
// Right now, the MediaRecorder hasn't dealt with multiple video track
// issues. So we just bind with the first video track. Bug 1276928 is
// the following.
videoTracks[0]->AddDirectListener(mEncoder->GetVideoSink());
}
}
// Try to use direct listeners if possible
if (domStream && domStream->GetInputStream()) { if (domStream && domStream->GetInputStream()) {
mInputStream = domStream->GetInputStream()->AsSourceStream(); mInputStream = domStream->GetInputStream()->AsSourceStream();
if (mInputStream) { if (mInputStream) {
mInputStream->AddDirectListener(mEncoder); mInputStream->AddDirectListener(mEncoder.get());
mEncoder->SetDirectConnect(true); mEncoder->SetDirectConnect(true);
} }
} }
@ -847,7 +864,7 @@ private:
{ {
if (mInputStream) { if (mInputStream) {
if (mEncoder) { if (mEncoder) {
mInputStream->RemoveDirectListener(mEncoder); mInputStream->RemoveDirectListener(mEncoder.get());
} }
mInputStream = nullptr; mInputStream = nullptr;
} }
@ -858,8 +875,13 @@ private:
mInputPorts.Clear(); mInputPorts.Clear();
if (mTrackUnionStream) { if (mTrackUnionStream) {
// Sometimes the MediaEncoder might be initialized fail and go to
// |CleanupStreams|. So the mEncoder might be a nullptr in this case.
if (mEncoder && mSelectedVideoTrackID != TRACK_NONE) {
mTrackUnionStream->RemoveVideoOutput(mEncoder->GetVideoSink(), mSelectedVideoTrackID);
}
if (mEncoder) { if (mEncoder) {
mTrackUnionStream->RemoveListener(mEncoder); mTrackUnionStream->RemoveListener(mEncoder.get());
} }
mTrackUnionStream->Destroy(); mTrackUnionStream->Destroy();
mTrackUnionStream = nullptr; mTrackUnionStream = nullptr;
@ -948,6 +970,7 @@ private:
// ExtractRunnable/DestroyRunnable will end the session. // ExtractRunnable/DestroyRunnable will end the session.
// Main thread only. // Main thread only.
bool mNeedSessionEndTask; bool mNeedSessionEndTask;
TrackID mSelectedVideoTrackID;
}; };
NS_IMPL_ISUPPORTS(MediaRecorder::Session, nsIObserver) NS_IMPL_ISUPPORTS(MediaRecorder::Session, nsIObserver)

Просмотреть файл

@ -33,6 +33,13 @@ mozilla::LazyLogModule gMediaEncoderLog("MediaEncoder");
namespace mozilla { namespace mozilla {
void
MediaStreamVideoRecorderSink::SetCurrentFrames(const VideoSegment& aSegment)
{
MOZ_ASSERT(mVideoEncoder);
mVideoEncoder->SetCurrentFrames(aSegment);
}
void void
MediaEncoder::SetDirectConnect(bool aConnected) MediaEncoder::SetDirectConnect(bool aConnected)
{ {
@ -53,8 +60,9 @@ MediaEncoder::NotifyRealtimeData(MediaStreamGraph* aGraph,
mAudioEncoder->NotifyQueuedTrackChanges(aGraph, aID, mAudioEncoder->NotifyQueuedTrackChanges(aGraph, aID,
aTrackOffset, aTrackEvents, aTrackOffset, aTrackEvents,
aRealtimeMedia); aRealtimeMedia);
} else if (mVideoEncoder &&
} else if (mVideoEncoder && aRealtimeMedia.GetType() == MediaSegment::VIDEO) { aRealtimeMedia.GetType() == MediaSegment::VIDEO &&
aTrackEvents != TrackEventCommand::TRACK_EVENT_NONE) {
mVideoEncoder->NotifyQueuedTrackChanges(aGraph, aID, mVideoEncoder->NotifyQueuedTrackChanges(aGraph, aID,
aTrackOffset, aTrackEvents, aTrackOffset, aTrackEvents,
aRealtimeMedia); aRealtimeMedia);
@ -141,7 +149,8 @@ MediaEncoder::NotifyEvent(MediaStreamGraph* aGraph,
already_AddRefed<MediaEncoder> already_AddRefed<MediaEncoder>
MediaEncoder::CreateEncoder(const nsAString& aMIMEType, uint32_t aAudioBitrate, MediaEncoder::CreateEncoder(const nsAString& aMIMEType, uint32_t aAudioBitrate,
uint32_t aVideoBitrate, uint32_t aBitrate, uint32_t aVideoBitrate, uint32_t aBitrate,
uint8_t aTrackTypes) uint8_t aTrackTypes,
TrackRate aTrackRate)
{ {
PROFILER_LABEL("MediaEncoder", "CreateEncoder", PROFILER_LABEL("MediaEncoder", "CreateEncoder",
js::ProfileEntry::Category::OTHER); js::ProfileEntry::Category::OTHER);
@ -164,7 +173,7 @@ MediaEncoder::CreateEncoder(const nsAString& aMIMEType, uint32_t aAudioBitrate,
audioEncoder = new OpusTrackEncoder(); audioEncoder = new OpusTrackEncoder();
NS_ENSURE_TRUE(audioEncoder, nullptr); NS_ENSURE_TRUE(audioEncoder, nullptr);
} }
videoEncoder = new VP8TrackEncoder(); videoEncoder = new VP8TrackEncoder(aTrackRate);
writer = new WebMWriter(aTrackTypes); writer = new WebMWriter(aTrackTypes);
NS_ENSURE_TRUE(writer, nullptr); NS_ENSURE_TRUE(writer, nullptr);
NS_ENSURE_TRUE(videoEncoder, nullptr); NS_ENSURE_TRUE(videoEncoder, nullptr);
@ -179,7 +188,7 @@ MediaEncoder::CreateEncoder(const nsAString& aMIMEType, uint32_t aAudioBitrate,
audioEncoder = new OmxAACAudioTrackEncoder(); audioEncoder = new OmxAACAudioTrackEncoder();
NS_ENSURE_TRUE(audioEncoder, nullptr); NS_ENSURE_TRUE(audioEncoder, nullptr);
} }
videoEncoder = new OmxVideoTrackEncoder(); videoEncoder = new OmxVideoTrackEncoder(aTrackRate);
writer = new ISOMediaWriter(aTrackTypes); writer = new ISOMediaWriter(aTrackTypes);
NS_ENSURE_TRUE(writer, nullptr); NS_ENSURE_TRUE(writer, nullptr);
NS_ENSURE_TRUE(videoEncoder, nullptr); NS_ENSURE_TRUE(videoEncoder, nullptr);

Просмотреть файл

@ -9,15 +9,32 @@
#include "mozilla/DebugOnly.h" #include "mozilla/DebugOnly.h"
#include "TrackEncoder.h" #include "TrackEncoder.h"
#include "ContainerWriter.h" #include "ContainerWriter.h"
#include "CubebUtils.h"
#include "MediaStreamGraph.h" #include "MediaStreamGraph.h"
#include "MediaStreamListener.h" #include "MediaStreamListener.h"
#include "nsAutoPtr.h" #include "nsAutoPtr.h"
#include "MediaStreamVideoSink.h"
#include "nsIMemoryReporter.h" #include "nsIMemoryReporter.h"
#include "mozilla/MemoryReporting.h" #include "mozilla/MemoryReporting.h"
#include "mozilla/Atomics.h" #include "mozilla/Atomics.h"
namespace mozilla { namespace mozilla {
class MediaStreamVideoRecorderSink : public MediaStreamVideoSink
{
public:
explicit MediaStreamVideoRecorderSink(VideoTrackEncoder* aEncoder)
: mVideoEncoder(aEncoder) {}
// MediaStreamVideoSink methods
virtual void SetCurrentFrames(const VideoSegment& aSegment) override;
virtual void ClearFrames() override {}
private:
virtual ~MediaStreamVideoRecorderSink() {}
VideoTrackEncoder* mVideoEncoder;
};
/** /**
* MediaEncoder is the framework of encoding module, it controls and manages * MediaEncoder is the framework of encoding module, it controls and manages
* procedures between ContainerWriter and TrackEncoder. ContainerWriter packs * procedures between ContainerWriter and TrackEncoder. ContainerWriter packs
@ -54,6 +71,7 @@ namespace mozilla {
*/ */
class MediaEncoder : public DirectMediaStreamListener class MediaEncoder : public DirectMediaStreamListener
{ {
friend class MediaStreamVideoRecorderSink;
public : public :
enum { enum {
ENCODE_METADDATA, ENCODE_METADDATA,
@ -72,6 +90,7 @@ public :
: mWriter(aWriter) : mWriter(aWriter)
, mAudioEncoder(aAudioEncoder) , mAudioEncoder(aAudioEncoder)
, mVideoEncoder(aVideoEncoder) , mVideoEncoder(aVideoEncoder)
, mVideoSink(new MediaStreamVideoRecorderSink(mVideoEncoder))
, mStartTime(TimeStamp::Now()) , mStartTime(TimeStamp::Now())
, mMIMEType(aMIMEType) , mMIMEType(aMIMEType)
, mSizeOfBuffer(0) , mSizeOfBuffer(0)
@ -155,7 +174,8 @@ public :
static already_AddRefed<MediaEncoder> CreateEncoder(const nsAString& aMIMEType, static already_AddRefed<MediaEncoder> CreateEncoder(const nsAString& aMIMEType,
uint32_t aAudioBitrate, uint32_t aVideoBitrate, uint32_t aAudioBitrate, uint32_t aVideoBitrate,
uint32_t aBitrate, uint32_t aBitrate,
uint8_t aTrackTypes = ContainerWriter::CREATE_AUDIO_TRACK); uint8_t aTrackTypes = ContainerWriter::CREATE_AUDIO_TRACK,
TrackRate aTrackRate = CubebUtils::PreferredSampleRate());
/** /**
* Encodes the raw track data and returns the final container data. Assuming * Encodes the raw track data and returns the final container data. Assuming
* it is called on a single worker thread. The buffer of container data is * it is called on a single worker thread. The buffer of container data is
@ -208,6 +228,10 @@ public :
*/ */
size_t SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf) const; size_t SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf) const;
MediaStreamVideoRecorderSink* GetVideoSink() {
return mVideoSink.get();
}
private: private:
// Get encoded data from trackEncoder and write to muxer // Get encoded data from trackEncoder and write to muxer
nsresult WriteEncodedDataToMuxer(TrackEncoder *aTrackEncoder); nsresult WriteEncodedDataToMuxer(TrackEncoder *aTrackEncoder);
@ -216,6 +240,7 @@ private:
nsAutoPtr<ContainerWriter> mWriter; nsAutoPtr<ContainerWriter> mWriter;
nsAutoPtr<AudioTrackEncoder> mAudioEncoder; nsAutoPtr<AudioTrackEncoder> mAudioEncoder;
nsAutoPtr<VideoTrackEncoder> mVideoEncoder; nsAutoPtr<VideoTrackEncoder> mVideoEncoder;
RefPtr<MediaStreamVideoRecorderSink> mVideoSink;
TimeStamp mStartTime; TimeStamp mStartTime;
nsString mMIMEType; nsString mMIMEType;
int64_t mSizeOfBuffer; int64_t mSizeOfBuffer;

Просмотреть файл

@ -26,8 +26,8 @@ namespace mozilla {
#define ENCODER_CONFIG_FRAME_RATE 30 // fps #define ENCODER_CONFIG_FRAME_RATE 30 // fps
#define GET_ENCODED_VIDEO_FRAME_TIMEOUT 100000 // microseconds #define GET_ENCODED_VIDEO_FRAME_TIMEOUT 100000 // microseconds
OmxVideoTrackEncoder::OmxVideoTrackEncoder() OmxVideoTrackEncoder::OmxVideoTrackEncoder(TrackRate aTrackRate)
: VideoTrackEncoder() : VideoTrackEncoder(aTrackRate)
{} {}
OmxVideoTrackEncoder::~OmxVideoTrackEncoder() OmxVideoTrackEncoder::~OmxVideoTrackEncoder()
@ -35,11 +35,10 @@ OmxVideoTrackEncoder::~OmxVideoTrackEncoder()
nsresult nsresult
OmxVideoTrackEncoder::Init(int aWidth, int aHeight, int aDisplayWidth, OmxVideoTrackEncoder::Init(int aWidth, int aHeight, int aDisplayWidth,
int aDisplayHeight, TrackRate aTrackRate) int aDisplayHeight)
{ {
mFrameWidth = aWidth; mFrameWidth = aWidth;
mFrameHeight = aHeight; mFrameHeight = aHeight;
mTrackRate = aTrackRate;
mDisplayWidth = aDisplayWidth; mDisplayWidth = aDisplayWidth;
mDisplayHeight = aDisplayHeight; mDisplayHeight = aDisplayHeight;

Просмотреть файл

@ -27,7 +27,7 @@ namespace mozilla {
class OmxVideoTrackEncoder: public VideoTrackEncoder class OmxVideoTrackEncoder: public VideoTrackEncoder
{ {
public: public:
OmxVideoTrackEncoder(); explicit OmxVideoTrackEncoder(TrackRate aTrackRate);
~OmxVideoTrackEncoder(); ~OmxVideoTrackEncoder();
already_AddRefed<TrackMetadataBase> GetMetadata() override; already_AddRefed<TrackMetadataBase> GetMetadata() override;
@ -36,8 +36,7 @@ public:
protected: protected:
nsresult Init(int aWidth, int aHeight, nsresult Init(int aWidth, int aHeight,
int aDisplayWidth, int aDisplayHeight, int aDisplayWidth, int aDisplayHeight) override;
TrackRate aTrackRate) override;
private: private:
nsAutoPtr<android::OMXVideoEncoder> mEncoder; nsAutoPtr<android::OMXVideoEncoder> mEncoder;

Просмотреть файл

@ -192,6 +192,46 @@ AudioTrackEncoder::SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf) cons
return mRawSegment.SizeOfExcludingThis(aMallocSizeOf); return mRawSegment.SizeOfExcludingThis(aMallocSizeOf);
} }
void
VideoTrackEncoder::Init(const VideoSegment& aSegment)
{
if (mInitialized) {
return;
}
mInitCounter++;
TRACK_LOG(LogLevel::Debug, ("Init the video encoder %d times", mInitCounter));
VideoSegment::ConstChunkIterator iter(aSegment);
while (!iter.IsEnded()) {
VideoChunk chunk = *iter;
if (!chunk.IsNull()) {
gfx::IntSize imgsize = chunk.mFrame.GetImage()->GetSize();
gfx::IntSize intrinsicSize = chunk.mFrame.GetIntrinsicSize();
nsresult rv = Init(imgsize.width, imgsize.height,
intrinsicSize.width, intrinsicSize.height);
if (NS_FAILED(rv)) {
LOG("[VideoTrackEncoder]: Fail to initialize the encoder!");
NotifyCancel();
}
break;
}
iter.Next();
}
}
void
VideoTrackEncoder::SetCurrentFrames(const VideoSegment& aSegment)
{
if (mCanceled) {
return;
}
Init(aSegment);
AppendVideoSegment(aSegment);
}
void void
VideoTrackEncoder::NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, VideoTrackEncoder::NotifyQueuedTrackChanges(MediaStreamGraph* aGraph,
TrackID aID, TrackID aID,
@ -203,40 +243,15 @@ VideoTrackEncoder::NotifyQueuedTrackChanges(MediaStreamGraph* aGraph,
return; return;
} }
if (!(aTrackEvents == TRACK_EVENT_CREATED ||
aTrackEvents == TRACK_EVENT_ENDED)) {
return;
}
const VideoSegment& video = static_cast<const VideoSegment&>(aQueuedMedia); const VideoSegment& video = static_cast<const VideoSegment&>(aQueuedMedia);
// Check and initialize parameters for codec encoder. // Check and initialize parameters for codec encoder.
if (!mInitialized) { Init(video);
mInitCounter++;
TRACK_LOG(LogLevel::Debug, ("Init the video encoder %d times", mInitCounter));
VideoSegment::ChunkIterator iter(const_cast<VideoSegment&>(video));
while (!iter.IsEnded()) {
VideoChunk chunk = *iter;
if (!chunk.IsNull()) {
gfx::IntSize imgsize = chunk.mFrame.GetImage()->GetSize();
gfx::IntSize intrinsicSize = chunk.mFrame.GetIntrinsicSize();
nsresult rv = Init(imgsize.width, imgsize.height,
intrinsicSize.width, intrinsicSize.height,
aGraph->GraphRate());
if (NS_FAILED(rv)) {
LOG("[VideoTrackEncoder]: Fail to initialize the encoder!");
NotifyCancel();
}
break;
}
iter.Next();
}
mNotInitDuration += aQueuedMedia.GetDuration();
if (!mInitialized &&
(mNotInitDuration / aGraph->GraphRate() > INIT_FAILED_DURATION) &&
mInitCounter > 1) {
LOG("[VideoTrackEncoder]: Initialize failed for 30s.");
NotifyEndOfStream();
return;
}
}
AppendVideoSegment(video); AppendVideoSegment(video);
@ -244,6 +259,7 @@ VideoTrackEncoder::NotifyQueuedTrackChanges(MediaStreamGraph* aGraph,
if (aTrackEvents == TrackEventCommand::TRACK_EVENT_ENDED) { if (aTrackEvents == TrackEventCommand::TRACK_EVENT_ENDED) {
LOG("[VideoTrackEncoder]: Receive TRACK_EVENT_ENDED ."); LOG("[VideoTrackEncoder]: Receive TRACK_EVENT_ENDED .");
NotifyEndOfStream(); NotifyEndOfStream();
mFirstFrame = true;
} }
} }
@ -266,6 +282,20 @@ VideoTrackEncoder::AppendVideoSegment(const VideoSegment& aSegment)
if ((mLastFrame != chunk.mFrame) || if ((mLastFrame != chunk.mFrame) ||
(mLastFrameDuration >= mTrackRate)) { (mLastFrameDuration >= mTrackRate)) {
RefPtr<layers::Image> image = chunk.mFrame.GetImage(); RefPtr<layers::Image> image = chunk.mFrame.GetImage();
// Fixme: see bug 1290777. We should remove the useage of duration here and
// in |GetEncodedTrack|.
StreamTime duration;
if (mFirstFrame)
{
duration = chunk.GetDuration();
mFirstFrame = false;
} else {
MOZ_ASSERT(chunk.mTimeStamp >= mLastFrameTimeStamp);
TimeDuration timeDuration = chunk.mTimeStamp - mLastFrameTimeStamp;
duration = SecondsToMediaTime(timeDuration.ToSeconds());
}
// Because we may get chunks with a null image (due to input blocking), // Because we may get chunks with a null image (due to input blocking),
// accumulate duration and give it to the next frame that arrives. // accumulate duration and give it to the next frame that arrives.
// Canonically incorrect - the duration should go to the previous frame // Canonically incorrect - the duration should go to the previous frame
@ -274,12 +304,13 @@ VideoTrackEncoder::AppendVideoSegment(const VideoSegment& aSegment)
// in with each frame. // in with each frame.
if (image) { if (image) {
mRawSegment.AppendFrame(image.forget(), mRawSegment.AppendFrame(image.forget(),
mLastFrameDuration, duration,
chunk.mFrame.GetIntrinsicSize(), chunk.mFrame.GetIntrinsicSize(),
PRINCIPAL_HANDLE_NONE, PRINCIPAL_HANDLE_NONE,
chunk.mFrame.GetForceBlack()); chunk.mFrame.GetForceBlack());
mLastFrameDuration = 0; mLastFrameDuration = 0;
} }
mLastFrameTimeStamp = chunk.mTimeStamp;
} }
mLastFrame.TakeFrom(&chunk.mFrame); mLastFrame.TakeFrom(&chunk.mFrame);
iter.Next(); iter.Next();
@ -299,7 +330,7 @@ VideoTrackEncoder::NotifyEndOfStream()
// encoder with default frame width, frame height, and track rate. // encoder with default frame width, frame height, and track rate.
if (!mCanceled && !mInitialized) { if (!mCanceled && !mInitialized) {
Init(DEFAULT_FRAME_WIDTH, DEFAULT_FRAME_HEIGHT, Init(DEFAULT_FRAME_WIDTH, DEFAULT_FRAME_HEIGHT,
DEFAULT_FRAME_WIDTH, DEFAULT_FRAME_HEIGHT, DEFAULT_TRACK_RATE); DEFAULT_FRAME_WIDTH, DEFAULT_FRAME_HEIGHT);
} }
ReentrantMonitorAutoEnter mon(mReentrantMonitor); ReentrantMonitorAutoEnter mon(mReentrantMonitor);

Просмотреть файл

@ -248,16 +248,18 @@ protected:
class VideoTrackEncoder : public TrackEncoder class VideoTrackEncoder : public TrackEncoder
{ {
public: public:
VideoTrackEncoder() explicit VideoTrackEncoder(TrackRate aTrackRate)
: TrackEncoder() : TrackEncoder()
, mFrameWidth(0) , mFrameWidth(0)
, mFrameHeight(0) , mFrameHeight(0)
, mDisplayWidth(0) , mDisplayWidth(0)
, mDisplayHeight(0) , mDisplayHeight(0)
, mTrackRate(0) , mTrackRate(aTrackRate)
, mTotalFrameDuration(0) , mTotalFrameDuration(0)
, mLastFrameDuration(0) , mLastFrameDuration(0)
, mVideoBitrate(0) , mVideoBitrate(0)
, mLastFrameTimeStamp(TimeStamp::Now())
, mFirstFrame(true)
{} {}
/** /**
@ -277,6 +279,18 @@ public:
{ {
mVideoBitrate = aBitrate; mVideoBitrate = aBitrate;
} }
void Init(const VideoSegment& aSegment);
void SetCurrentFrames(const VideoSegment& aSegment);
StreamTime SecondsToMediaTime(double aS) const
{
NS_ASSERTION(0 <= aS && aS <= TRACK_TICKS_MAX/TRACK_RATE_MAX,
"Bad seconds");
return mTrackRate * aS;
}
protected: protected:
/** /**
* Initialized the video encoder. In order to collect the value of width and * Initialized the video encoder. In order to collect the value of width and
@ -286,7 +300,7 @@ protected:
* and this method is called on the MediaStramGraph thread. * and this method is called on the MediaStramGraph thread.
*/ */
virtual nsresult Init(int aWidth, int aHeight, int aDisplayWidth, virtual nsresult Init(int aWidth, int aHeight, int aDisplayWidth,
int aDisplayHeight, TrackRate aTrackRate) = 0; int aDisplayHeight) = 0;
/** /**
* Appends source video frames to mRawSegment. We only append the source chunk * Appends source video frames to mRawSegment. We only append the source chunk
@ -345,6 +359,10 @@ protected:
VideoSegment mRawSegment; VideoSegment mRawSegment;
uint32_t mVideoBitrate; uint32_t mVideoBitrate;
private:
TimeStamp mLastFrameTimeStamp;
bool mFirstFrame;
}; };
} // namespace mozilla } // namespace mozilla

Просмотреть файл

@ -28,8 +28,8 @@ LazyLogModule gVP8TrackEncoderLog("VP8TrackEncoder");
using namespace mozilla::gfx; using namespace mozilla::gfx;
using namespace mozilla::layers; using namespace mozilla::layers;
VP8TrackEncoder::VP8TrackEncoder() VP8TrackEncoder::VP8TrackEncoder(TrackRate aTrackRate)
: VideoTrackEncoder() : VideoTrackEncoder(aTrackRate)
, mEncodedFrameDuration(0) , mEncodedFrameDuration(0)
, mEncodedTimestamp(0) , mEncodedTimestamp(0)
, mRemainingTicks(0) , mRemainingTicks(0)
@ -53,16 +53,14 @@ VP8TrackEncoder::~VP8TrackEncoder()
nsresult nsresult
VP8TrackEncoder::Init(int32_t aWidth, int32_t aHeight, int32_t aDisplayWidth, VP8TrackEncoder::Init(int32_t aWidth, int32_t aHeight, int32_t aDisplayWidth,
int32_t aDisplayHeight,TrackRate aTrackRate) int32_t aDisplayHeight)
{ {
if (aWidth < 1 || aHeight < 1 || aDisplayWidth < 1 || aDisplayHeight < 1 if (aWidth < 1 || aHeight < 1 || aDisplayWidth < 1 || aDisplayHeight < 1) {
|| aTrackRate <= 0) {
return NS_ERROR_FAILURE; return NS_ERROR_FAILURE;
} }
ReentrantMonitorAutoEnter mon(mReentrantMonitor); ReentrantMonitorAutoEnter mon(mReentrantMonitor);
mTrackRate = aTrackRate;
mEncodedFrameRate = DEFAULT_ENCODE_FRAMERATE; mEncodedFrameRate = DEFAULT_ENCODE_FRAMERATE;
mEncodedFrameDuration = mTrackRate / mEncodedFrameRate; mEncodedFrameDuration = mTrackRate / mEncodedFrameRate;
mFrameWidth = aWidth; mFrameWidth = aWidth;

Просмотреть файл

@ -29,7 +29,7 @@ class VP8TrackEncoder : public VideoTrackEncoder
SKIP_FRAME, // Skip the next frame. SKIP_FRAME, // Skip the next frame.
}; };
public: public:
VP8TrackEncoder(); explicit VP8TrackEncoder(TrackRate aTrackRate);
virtual ~VP8TrackEncoder(); virtual ~VP8TrackEncoder();
already_AddRefed<TrackMetadataBase> GetMetadata() final override; already_AddRefed<TrackMetadataBase> GetMetadata() final override;
@ -38,8 +38,7 @@ public:
protected: protected:
nsresult Init(int32_t aWidth, int32_t aHeight, nsresult Init(int32_t aWidth, int32_t aHeight,
int32_t aDisplayWidth, int32_t aDisplayHeight, int32_t aDisplayWidth, int32_t aDisplayHeight) final override;
TrackRate aTrackRate) final override;
private: private:
// Calculate the target frame's encoded duration. // Calculate the target frame's encoded duration.

Просмотреть файл

@ -175,22 +175,23 @@ struct InitParam {
bool mShouldSucceed; // This parameter should cause success or fail result bool mShouldSucceed; // This parameter should cause success or fail result
int mWidth; // frame width int mWidth; // frame width
int mHeight; // frame height int mHeight; // frame height
mozilla::TrackRate mTrackRate; // track rate. 90K is the most commond track rate.
}; };
class TestVP8TrackEncoder: public VP8TrackEncoder class TestVP8TrackEncoder: public VP8TrackEncoder
{ {
public: public:
explicit TestVP8TrackEncoder(TrackRate aTrackRate = 90000)
: VP8TrackEncoder(aTrackRate) {}
::testing::AssertionResult TestInit(const InitParam &aParam) ::testing::AssertionResult TestInit(const InitParam &aParam)
{ {
nsresult result = Init(aParam.mWidth, aParam.mHeight, aParam.mWidth, aParam.mHeight, aParam.mTrackRate); nsresult result = Init(aParam.mWidth, aParam.mHeight, aParam.mWidth, aParam.mHeight);
if (((NS_FAILED(result) && aParam.mShouldSucceed)) || (NS_SUCCEEDED(result) && !aParam.mShouldSucceed)) if (((NS_FAILED(result) && aParam.mShouldSucceed)) || (NS_SUCCEEDED(result) && !aParam.mShouldSucceed))
{ {
return ::testing::AssertionFailure() return ::testing::AssertionFailure()
<< " width = " << aParam.mWidth << " width = " << aParam.mWidth
<< " height = " << aParam.mHeight << " height = " << aParam.mHeight;
<< " TrackRate = " << aParam.mTrackRate << ".";
} }
else else
{ {
@ -204,17 +205,15 @@ TEST(VP8VideoTrackEncoder, Initialization)
{ {
InitParam params[] = { InitParam params[] = {
// Failure cases. // Failure cases.
{ false, 640, 480, 0 }, // Trackrate should be larger than 1. { false, 0, 0}, // Height/ width should be larger than 1.
{ false, 640, 480, -1 }, // Trackrate should be larger than 1. { false, 0, 1}, // Height/ width should be larger than 1.
{ false, 0, 0, 90000 }, // Height/ width should be larger than 1. { false, 1, 0}, // Height/ width should be larger than 1.
{ false, 0, 1, 90000 }, // Height/ width should be larger than 1.
{ false, 1, 0, 90000}, // Height/ width should be larger than 1.
// Success cases // Success cases
{ true, 640, 480, 90000}, // Standard VGA { true, 640, 480}, // Standard VGA
{ true, 800, 480, 90000}, // Standard WVGA { true, 800, 480}, // Standard WVGA
{ true, 960, 540, 90000}, // Standard qHD { true, 960, 540}, // Standard qHD
{ true, 1280, 720, 90000} // Standard HD { true, 1280, 720} // Standard HD
}; };
for (size_t i = 0; i < ArrayLength(params); i++) for (size_t i = 0; i < ArrayLength(params); i++)
@ -229,10 +228,10 @@ TEST(VP8VideoTrackEncoder, FetchMetaData)
{ {
InitParam params[] = { InitParam params[] = {
// Success cases // Success cases
{ true, 640, 480, 90000}, // Standard VGA { true, 640, 480}, // Standard VGA
{ true, 800, 480, 90000}, // Standard WVGA { true, 800, 480}, // Standard WVGA
{ true, 960, 540, 90000}, // Standard qHD { true, 960, 540}, // Standard qHD
{ true, 1280, 720, 90000} // Standard HD { true, 1280, 720} // Standard HD
}; };
for (size_t i = 0; i < ArrayLength(params); i++) for (size_t i = 0; i < ArrayLength(params); i++)
@ -254,7 +253,7 @@ TEST(VP8VideoTrackEncoder, FrameEncode)
{ {
// Initiate VP8 encoder // Initiate VP8 encoder
TestVP8TrackEncoder encoder; TestVP8TrackEncoder encoder;
InitParam param = {true, 640, 480, 90000}; InitParam param = {true, 640, 480};
encoder.TestInit(param); encoder.TestInit(param);
// Create YUV images as source. // Create YUV images as source.
@ -288,7 +287,7 @@ TEST(VP8VideoTrackEncoder, EncodeComplete)
{ {
// Initiate VP8 encoder // Initiate VP8 encoder
TestVP8TrackEncoder encoder; TestVP8TrackEncoder encoder;
InitParam param = {true, 640, 480, 90000}; InitParam param = {true, 640, 480};
encoder.TestInit(param); encoder.TestInit(param);
// track end notification. // track end notification.

Просмотреть файл

@ -28,11 +28,13 @@ public:
class WebMVP8TrackEncoder: public VP8TrackEncoder class WebMVP8TrackEncoder: public VP8TrackEncoder
{ {
public: public:
explicit WebMVP8TrackEncoder(TrackRate aTrackRate = 90000)
: VP8TrackEncoder(aTrackRate) {}
bool TestVP8Creation(int32_t aWidth, int32_t aHeight, int32_t aDisplayWidth, bool TestVP8Creation(int32_t aWidth, int32_t aHeight, int32_t aDisplayWidth,
int32_t aDisplayHeight, TrackRate aTrackRate) int32_t aDisplayHeight)
{ {
if (NS_SUCCEEDED(Init(aWidth, aHeight, aDisplayWidth, aDisplayHeight, if (NS_SUCCEEDED(Init(aWidth, aHeight, aDisplayWidth, aDisplayHeight))) {
aTrackRate))) {
return true; return true;
} }
return false; return false;
@ -60,7 +62,7 @@ public:
int32_t aDisplayHeight,TrackRate aTrackRate) { int32_t aDisplayHeight,TrackRate aTrackRate) {
WebMVP8TrackEncoder vp8Encoder; WebMVP8TrackEncoder vp8Encoder;
EXPECT_TRUE(vp8Encoder.TestVP8Creation(aWidth, aHeight, aDisplayWidth, EXPECT_TRUE(vp8Encoder.TestVP8Creation(aWidth, aHeight, aDisplayWidth,
aDisplayHeight, aTrackRate)); aDisplayHeight));
RefPtr<TrackMetadataBase> vp8Meta = vp8Encoder.GetMetadata(); RefPtr<TrackMetadataBase> vp8Meta = vp8Encoder.GetMetadata();
SetMetadata(vp8Meta); SetMetadata(vp8Meta);
} }