From a0fcf04b6738d21f8b483cf94b0cde65b93e6298 Mon Sep 17 00:00:00 2001 From: Andreas Pehrson Date: Wed, 13 May 2015 14:04:41 +0800 Subject: [PATCH] Bug 1032848 - Part 2: Implement HTMLCanvasElement::CaptureStream. r=mt, r=jesup, r=jgilbert, r=gwright --HG-- extra : transplant_source : o%13%2CuRJ2%21%B0%AE%5D%F0%B3%11%B30Y%5B%A3N --- dom/canvas/WebGLContext.h | 2 + dom/html/HTMLCanvasElement.cpp | 43 +++- dom/media/CanvasCaptureMediaStream.cpp | 269 ++++++++++++++++++++++++- dom/media/CanvasCaptureMediaStream.h | 60 ++++++ 4 files changed, 370 insertions(+), 4 deletions(-) diff --git a/dom/canvas/WebGLContext.h b/dom/canvas/WebGLContext.h index 2b7ef2603ca7..0b0c41922b1d 100644 --- a/dom/canvas/WebGLContext.h +++ b/dom/canvas/WebGLContext.h @@ -310,6 +310,8 @@ public: bool IsPremultAlpha() const { return mOptions.premultipliedAlpha; } + bool IsPreservingDrawingBuffer() const { return mOptions.preserveDrawingBuffer; } + bool PresentScreenBuffer(); // a number that increments every time we have an event that causes diff --git a/dom/html/HTMLCanvasElement.cpp b/dom/html/HTMLCanvasElement.cpp index 6e4c1c349efb..85992ced0647 100644 --- a/dom/html/HTMLCanvasElement.cpp +++ b/dom/html/HTMLCanvasElement.cpp @@ -410,8 +410,47 @@ already_AddRefed HTMLCanvasElement::CaptureStream(const Optional& aFrameRate, ErrorResult& aRv) { - aRv.Throw(NS_ERROR_NOT_IMPLEMENTED); - return nullptr; + if (IsWriteOnly()) { + aRv.Throw(NS_ERROR_DOM_SECURITY_ERR); + return nullptr; + } + + nsIDOMWindow* window = OwnerDoc()->GetInnerWindow(); + if (!window) { + aRv.Throw(NS_ERROR_FAILURE); + return nullptr; + } + + if (!mCurrentContext) { + aRv.Throw(NS_ERROR_NOT_INITIALIZED); + return nullptr; + } + + if (mCurrentContextType != CanvasContextType::Canvas2D) { + WebGLContext* gl = static_cast(mCurrentContext.get()); + if (!gl->IsPreservingDrawingBuffer()) { + aRv.Throw(NS_ERROR_FAILURE); + return nullptr; + } + } + + nsRefPtr stream = + CanvasCaptureMediaStream::CreateSourceStream(window, this); + if (!stream) { + aRv.Throw(NS_ERROR_FAILURE); + return nullptr; + } + + nsRefPtr principal = NodePrincipal(); + stream->CombineWithPrincipal(principal); + + TrackID videoTrackId = 1; + nsresult rv = stream->Init(aFrameRate, videoTrackId); + if (NS_FAILED(rv)) { + aRv.Throw(rv); + return nullptr; + } + return stream.forget(); } nsresult diff --git a/dom/media/CanvasCaptureMediaStream.cpp b/dom/media/CanvasCaptureMediaStream.cpp index 3433f42ab5db..c2e25736bfed 100644 --- a/dom/media/CanvasCaptureMediaStream.cpp +++ b/dom/media/CanvasCaptureMediaStream.cpp @@ -5,6 +5,7 @@ #include "CanvasCaptureMediaStream.h" #include "DOMMediaStream.h" +#include "gfxPlatform.h" #include "ImageContainer.h" #include "MediaStreamGraph.h" #include "mozilla/Mutex.h" @@ -21,6 +22,257 @@ using namespace mozilla::gfx; namespace mozilla { namespace dom { +class OutputStreamDriver::StreamListener : public MediaStreamListener +{ +public: + explicit StreamListener(OutputStreamDriver* aDriver, + SourceMediaStream* aSourceStream) + : mSourceStream(aSourceStream) + , mMutex("CanvasCaptureMediaStream::OSD::StreamListener") + , mDriver(aDriver) + { + MOZ_ASSERT(mDriver); + MOZ_ASSERT(mSourceStream); + } + + void Forget() { + MOZ_ASSERT(NS_IsMainThread()); + + MutexAutoLock lock(mMutex); + mDriver = nullptr; + } + + virtual void NotifyPull(MediaStreamGraph* aGraph, StreamTime aDesiredTime) override + { + // Called on the MediaStreamGraph thread. + + MutexAutoLock lock(mMutex); + if (mDriver) { + mDriver->NotifyPull(aDesiredTime); + } else { + // The DOM stream is dead, let's end it + mSourceStream->EndAllTrackAndFinish(); + } + } + +protected: + ~StreamListener() { } + +private: + nsRefPtr mSourceStream; + + // The below members are protected by mMutex. + Mutex mMutex; + // This is a raw pointer to avoid a reference cycle with OutputStreamDriver. + // Accessed on main and MediaStreamGraph threads, set on main thread. + OutputStreamDriver* mDriver; +}; + +OutputStreamDriver::OutputStreamDriver(CanvasCaptureMediaStream* aDOMStream, + const TrackID& aTrackId) + : mDOMStream(aDOMStream) + , mSourceStream(nullptr) + , mStarted(false) + , mStreamListener(nullptr) + , mTrackId(aTrackId) + , mMutex("CanvasCaptureMediaStream::OutputStreamDriver") + , mImage(nullptr) +{ + MOZ_ASSERT(mDOMStream); +} + +OutputStreamDriver::~OutputStreamDriver() +{ + if (mStreamListener) { + // MediaStreamGraph will keep the listener alive until it can finish the + // stream on the next NotifyPull(). + mStreamListener->Forget(); + } +} + +nsresult +OutputStreamDriver::Start() +{ + if (mStarted) { + return NS_ERROR_ALREADY_INITIALIZED; + } + + MOZ_ASSERT(mDOMStream); + + mDOMStream->CreateDOMTrack(mTrackId, MediaSegment::VIDEO); + + mSourceStream = mDOMStream->GetStream()->AsSourceStream(); + MOZ_ASSERT(mSourceStream); + + mStreamListener = new StreamListener(this, mSourceStream); + mSourceStream->AddListener(mStreamListener); + mSourceStream->AddTrack(mTrackId, 0, new VideoSegment()); + mSourceStream->AdvanceKnownTracksTime(STREAM_TIME_MAX); + mSourceStream->SetPullEnabled(true); + + // Run StartInternal() in stable state to allow it to directly capture a frame + nsCOMPtr runnable = + NS_NewRunnableMethod(this, &OutputStreamDriver::StartInternal); + nsCOMPtr appShell = do_GetService(kAppShellCID); + appShell->RunInStableState(runnable); + + mStarted = true; + return NS_OK; +} + +void +OutputStreamDriver::ForgetDOMStream() +{ + if (mStreamListener) { + mStreamListener->Forget(); + } + mDOMStream = nullptr; +} + +void +OutputStreamDriver::AppendToTrack(StreamTime aDuration) +{ + MOZ_ASSERT(mSourceStream); + + MutexAutoLock lock(mMutex); + + nsRefPtr image = mImage; + IntSize size = image ? image->GetSize() : IntSize(0, 0); + VideoSegment segment; + segment.AppendFrame(image.forget(), aDuration, size); + + mSourceStream->AppendToTrack(mTrackId, &segment); +} + +void +OutputStreamDriver::NotifyPull(StreamTime aDesiredTime) +{ + StreamTime delta = aDesiredTime - mSourceStream->GetEndOfAppendedData(mTrackId); + if (delta > 0) { + // nullptr images are allowed + AppendToTrack(delta); + } +} + +void +OutputStreamDriver::SetImage(Image* aImage) +{ + MutexAutoLock lock(mMutex); + mImage = aImage; +} + +// ---------------------------------------------------------------------- + +class TimerDriver : public OutputStreamDriver + , public nsITimerCallback +{ +public: + explicit TimerDriver(CanvasCaptureMediaStream* aDOMStream, + const double& aFPS, + const TrackID& aTrackId) + : OutputStreamDriver(aDOMStream, aTrackId) + , mFPS(aFPS) + , mTimer(nullptr) + { + } + + void ForgetDOMStream() override + { + if (mTimer) { + mTimer->Cancel(); + mTimer = nullptr; + } + OutputStreamDriver::ForgetDOMStream(); + } + + nsresult + TakeSnapshot() + { + // mDOMStream can't be killed while we're on main thread + MOZ_ASSERT(NS_IsMainThread()); + MOZ_ASSERT(DOMStream()); + + if (!DOMStream()->Canvas()) { + // DOMStream's canvas pointer was garbage collected. We can abort now. + return NS_ERROR_NOT_AVAILABLE; + } + MOZ_ASSERT(DOMStream()->Canvas()); + + if (DOMStream()->Canvas()->IsWriteOnly()) { + return NS_ERROR_DOM_SECURITY_ERR; + } + + // Pass `nullptr` to force alpha-premult. + RefPtr snapshot = DOMStream()->Canvas()->GetSurfaceSnapshot(nullptr); + if (!snapshot) { + return NS_ERROR_FAILURE; + } + + RefPtr opt = gfxPlatform::GetPlatform() + ->ScreenReferenceDrawTarget()->OptimizeSourceSurface(snapshot); + if (!opt) { + return NS_ERROR_FAILURE; + } + + CairoImage::Data imageData; + imageData.mSize = opt->GetSize(); + imageData.mSourceSurface = opt; + + RefPtr image = new layers::CairoImage(); + image->SetData(imageData); + + SetImage(image); + return NS_OK; + } + + NS_IMETHODIMP + Notify(nsITimer* aTimer) + { + nsresult rv = TakeSnapshot(); + if (NS_FAILED(rv)) { + aTimer->Cancel(); + } + return rv; + } + + virtual void RequestFrame() override + { + TakeSnapshot(); + } + + NS_DECL_ISUPPORTS_INHERITED + +protected: + virtual ~TimerDriver() {} + + virtual void StartInternal() override + { + // Always capture at least one frame. + DebugOnly rv = TakeSnapshot(); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + + if (mFPS == 0.0) { + return; + } + + mTimer = do_CreateInstance(NS_TIMER_CONTRACTID); + if (!mTimer) { + return; + } + mTimer->InitWithCallback(this, int(1000 / mFPS), nsITimer::TYPE_REPEATING_SLACK); + } + +private: + const double mFPS; + nsCOMPtr mTimer; +}; + +NS_IMPL_ADDREF_INHERITED(TimerDriver, OutputStreamDriver) +NS_IMPL_RELEASE_INHERITED(TimerDriver, OutputStreamDriver) +NS_IMPL_QUERY_INTERFACE(TimerDriver, nsITimerCallback) + +// ---------------------------------------------------------------------- + NS_IMPL_CYCLE_COLLECTION_INHERITED(CanvasCaptureMediaStream, DOMMediaStream, mCanvas) @@ -32,11 +284,15 @@ NS_INTERFACE_MAP_END_INHERITING(DOMMediaStream) CanvasCaptureMediaStream::CanvasCaptureMediaStream(HTMLCanvasElement* aCanvas) : mCanvas(aCanvas) + , mOutputStreamDriver(nullptr) { } CanvasCaptureMediaStream::~CanvasCaptureMediaStream() { + if (mOutputStreamDriver) { + mOutputStreamDriver->ForgetDOMStream(); + } } JSObject* @@ -48,6 +304,9 @@ CanvasCaptureMediaStream::WrapObject(JSContext* aCx, JS::Handle aGive void CanvasCaptureMediaStream::RequestFrame() { + if (mOutputStreamDriver) { + mOutputStreamDriver->RequestFrame(); + } } nsresult @@ -55,11 +314,17 @@ CanvasCaptureMediaStream::Init(const dom::Optional& aFPS, const TrackID& aTrackId) { if (!aFPS.WasPassed()) { - return NS_ERROR_NOT_IMPLEMENTED; + // TODO (Bug 1152298): Implement a real AutoDriver. + // We use a 30FPS TimerDriver for now. + mOutputStreamDriver = new TimerDriver(this, 30.0, aTrackId); } else if (aFPS.Value() < 0) { return NS_ERROR_ILLEGAL_VALUE; + } else { + // Cap frame rate to 60 FPS for sanity + double fps = std::min(60.0, aFPS.Value()); + mOutputStreamDriver = new TimerDriver(this, fps, aTrackId); } - return NS_ERROR_NOT_IMPLEMENTED; + return mOutputStreamDriver->Start(); } already_AddRefed diff --git a/dom/media/CanvasCaptureMediaStream.h b/dom/media/CanvasCaptureMediaStream.h index de99b1774ad1..bb4aa13ba513 100644 --- a/dom/media/CanvasCaptureMediaStream.h +++ b/dom/media/CanvasCaptureMediaStream.h @@ -8,10 +8,69 @@ namespace mozilla { class DOMMediaStream; +class MediaStreamListener; +class SourceMediaStream; + +namespace layers { +class Image; +} namespace dom { +class CanvasCaptureMediaStream; class HTMLCanvasElement; +class OutputStreamDriver +{ +public: + OutputStreamDriver(CanvasCaptureMediaStream* aDOMStream, + const TrackID& aTrackId); + + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(OutputStreamDriver); + + nsresult Start(); + + virtual void ForgetDOMStream(); + + virtual void RequestFrame() { } + + CanvasCaptureMediaStream* DOMStream() const { return mDOMStream; } + +protected: + virtual ~OutputStreamDriver(); + class StreamListener; + + /* + * Appends mImage to video track for the desired duration. + */ + void AppendToTrack(StreamTime aDuration); + void NotifyPull(StreamTime aDesiredTime); + + /* + * Sub classes can SetImage() to update the image being appended to the + * output stream. It will be appended on the next NotifyPull from MSG. + */ + void SetImage(layers::Image* aImage); + + /* + * Called in main thread stable state to initialize sub classes. + */ + virtual void StartInternal() = 0; + +private: + // This is a raw pointer to avoid a reference cycle between OutputStreamDriver + // and CanvasCaptureMediaStream. ForgetDOMStream() will be called by + // ~CanvasCaptureMediaStream() to make sure we don't do anything illegal. + CanvasCaptureMediaStream* mDOMStream; + nsRefPtr mSourceStream; + bool mStarted; + nsRefPtr mStreamListener; + const TrackID mTrackId; + + // The below members are protected by mMutex. + Mutex mMutex; + nsRefPtr mImage; +}; + class CanvasCaptureMediaStream: public DOMMediaStream { public: @@ -40,6 +99,7 @@ protected: private: nsRefPtr mCanvas; + nsRefPtr mOutputStreamDriver; }; } // namespace dom