diff --git a/content/media/webrtc/MediaEngineWebRTCAudio.cpp b/content/media/webrtc/MediaEngineWebRTCAudio.cpp index 03a36cf598b6..720c85be02b4 100644 --- a/content/media/webrtc/MediaEngineWebRTCAudio.cpp +++ b/content/media/webrtc/MediaEngineWebRTCAudio.cpp @@ -139,6 +139,13 @@ MediaEngineWebRTCAudioSource::NotifyPull(MediaStreamGraph* aGraph, StreamTime aDesiredTime) { // Ignore - we push audio data +#ifdef DEBUG + static TrackTicks mLastEndTime = 0; + TrackTicks target = TimeToTicksRoundUp(SAMPLE_FREQUENCY, aDesiredTime); + TrackTicks delta = target - mLastEndTime; + LOG(("Audio:NotifyPull: target %lu, delta %lu",(uint32_t) target, (uint32_t) delta)); + mLastEndTime = target; +#endif } nsresult diff --git a/content/media/webrtc/MediaEngineWebRTCVideo.cpp b/content/media/webrtc/MediaEngineWebRTCVideo.cpp index 42b549e6d05f..4b39b1731c3c 100644 --- a/content/media/webrtc/MediaEngineWebRTCVideo.cpp +++ b/content/media/webrtc/MediaEngineWebRTCVideo.cpp @@ -251,6 +251,7 @@ MediaEngineWebRTCVideoSource::Start(SourceMediaStream* aStream, TrackID aID) mSource->AddTrack(aID, USECS_PER_S, 0, new VideoSegment()); mSource->AdvanceKnownTracksTime(STREAM_TIME_MAX); mLastEndTime = 0; + mState = kStarted; error = mViERender->AddRenderer(mCaptureIndex, webrtc::kVideoI420, (webrtc::ExternalRenderer*)this); if (error == -1) { @@ -266,7 +267,6 @@ MediaEngineWebRTCVideoSource::Start(SourceMediaStream* aStream, TrackID aID) return NS_ERROR_FAILURE; } - mState = kStarted; return NS_OK; } diff --git a/dom/media/MediaManager.cpp b/dom/media/MediaManager.cpp index ac1caf772749..21af6f724e36 100644 --- a/dom/media/MediaManager.cpp +++ b/dom/media/MediaManager.cpp @@ -62,9 +62,12 @@ public: nsCOMPtr success(mSuccess); nsCOMPtr error(mError); - WindowTable* activeWindows = MediaManager::Get()->GetActiveWindows(); - if (activeWindows->Get(mWindowID)) { - error->OnError(mErrorMsg); + { + MutexAutoLock lock(MediaManager::Get()->GetMutex()); + WindowTable* activeWindows = MediaManager::Get()->GetActiveWindows(); + if (activeWindows->Get(mWindowID)) { + error->OnError(mErrorMsg); + } } return NS_OK; } @@ -103,10 +106,13 @@ public: nsCOMPtr success(mSuccess); nsCOMPtr error(mError); - WindowTable* activeWindows = MediaManager::Get()->GetActiveWindows(); - if (activeWindows->Get(mWindowID)) { - // XPConnect is a magical unicorn. - success->OnSuccess(mFile); + { + MutexAutoLock lock(MediaManager::Get()->GetMutex()); + WindowTable* activeWindows = MediaManager::Get()->GetActiveWindows(); + if (activeWindows->Get(mWindowID)) { + // XPConnect is a magical unicorn. + success->OnSuccess(mFile); + } } return NS_OK; } @@ -209,7 +215,7 @@ MediaDevice::GetSource() * Note that the various GetUserMedia Runnable classes currently allow for * two streams. If we ever need to support getting more than two streams * at once, we could convert everything to nsTArray >'s, - * though that would complicate the constructors some. Currently the + * though that would complicate the constructors some. Currently the * GetUserMedia spec does not allow for more than 2 streams to be obtained in * one call, to simplify handling of constraints. */ @@ -247,38 +253,54 @@ public: nsPIDOMWindow *window = static_cast (nsGlobalWindow::GetInnerWindowWithId(mWindowID)); WindowTable* activeWindows = MediaManager::Get()->GetActiveWindows(); + { + MutexAutoLock lock(MediaManager::Get()->GetMutex()); - if (!stream) { - if (activeWindows->Get(mWindowID)) { - nsCOMPtr error(mError); - LOG(("Returning error for getUserMedia() - no stream")); - error->OnError(NS_LITERAL_STRING("NO_STREAM")); + if (!stream) { + if (activeWindows->Get(mWindowID)) { + nsCOMPtr error(mError); + LOG(("Returning error for getUserMedia() - no stream")); + error->OnError(NS_LITERAL_STRING("NO_STREAM")); + } + return NS_OK; } - return NS_OK; } - if (window && window->GetExtantDoc()) { stream->CombineWithPrincipal(window->GetExtantDoc()->NodePrincipal()); } + // Ensure there's a thread for gum to proxy to off main thread + nsIThread *mediaThread = MediaManager::GetThread(); + // Add our listener. We'll call Start() on the source when get a callback // that the MediaStream has started consuming. The listener is freed // when the page is invalidated (on navigation or close). GetUserMediaCallbackMediaStreamListener* listener = - new GetUserMediaCallbackMediaStreamListener(stream, mAudioSource, + new GetUserMediaCallbackMediaStreamListener(mediaThread, stream, + mAudioSource, mVideoSource); stream->GetStream()->AddListener(listener); // No need for locking because we always do this in the main thread. mListeners->AppendElement(listener); + // Dispatch to the media thread to ask it to start the sources, + // because that can take a while + nsRefPtr runnable( + new MediaOperationRunnable(MEDIA_START, stream, + mAudioSource, mVideoSource)); + mediaThread->Dispatch(runnable, NS_DISPATCH_NORMAL); + // We're in the main thread, so no worries here either. nsCOMPtr success(mSuccess); nsCOMPtr error(mError); - if (activeWindows->Get(mWindowID)) { - LOG(("Returning success for getUserMedia()")); - success->OnSuccess(stream); + { + MutexAutoLock lock(MediaManager::Get()->GetMutex()); + if (activeWindows->Get(mWindowID)) { + LOG(("Returning success for getUserMedia()")); + success->OnSuccess(stream); + } } return NS_OK; @@ -312,7 +334,7 @@ public: GetUserMediaRunnable(bool aAudio, bool aVideo, bool aPicture, already_AddRefed aSuccess, already_AddRefed aError, - StreamListeners* aListeners, uint64_t aWindowID, + StreamListeners* aListeners, uint64_t aWindowID, MediaDevice* aAudioDevice, MediaDevice* aVideoDevice) : mAudio(aAudio) , mVideo(aVideo) @@ -326,7 +348,7 @@ public: { if (mAudio) { mAudioDevice = aAudioDevice; - } + } if (mVideo) { mVideoDevice = aVideoDevice; } @@ -759,49 +781,53 @@ MediaManager::GetUserMedia(bool aPrivileged, nsPIDOMWindow* aWindow, // Store the WindowID in a hash table and mark as active. The entry is removed // when this window is closed or navigated away from. uint64_t windowID = aWindow->WindowID(); - StreamListeners* listeners = mActiveWindows.Get(windowID); - if (!listeners) { - listeners = new StreamListeners; - mActiveWindows.Put(windowID, listeners); - } - - // Developer preference for turning off permission check. - if (Preferences::GetBool("media.navigator.permission.disabled", false)) { - aPrivileged = true; - } - - /** - * Pass runnables along to GetUserMediaRunnable so it can add the - * MediaStreamListener to the runnable list. The last argument can - * optionally be a MediaDevice object, which should provided if one was - * selected by the user via the UI, or was provided by privileged code - * via the device: attribute via nsIMediaStreamOptions. - * - * If a fake stream was requested, we force the use of the default backend. - */ nsRefPtr gUMRunnable; - if (fake) { - // Fake stream from default backend. - gUMRunnable = new GetUserMediaRunnable( - audio, video, onSuccess.forget(), onError.forget(), listeners, - windowID, new MediaEngineDefault() - ); - } else if (audiodevice || videodevice) { - // Stream from provided device. - gUMRunnable = new GetUserMediaRunnable( - audio, video, picture, onSuccess.forget(), onError.forget(), listeners, - windowID, - static_cast(audiodevice.get()), - static_cast(videodevice.get()) - ); - } else { - // Stream from default device from WebRTC backend. - gUMRunnable = new GetUserMediaRunnable( - audio, video, picture, onSuccess.forget(), onError.forget(), listeners, - windowID - ); + { + MutexAutoLock lock(mMutex); + StreamListeners* listeners = mActiveWindows.Get(windowID); + if (!listeners) { + listeners = new StreamListeners; + mActiveWindows.Put(windowID, listeners); + } + + // Developer preference for turning off permission check. + if (Preferences::GetBool("media.navigator.permission.disabled", false)) { + aPrivileged = true; + } + + /** + * Pass runnables along to GetUserMediaRunnable so it can add the + * MediaStreamListener to the runnable list. The last argument can + * optionally be a MediaDevice object, which should provided if one was + * selected by the user via the UI, or was provided by privileged code + * via the device: attribute via nsIMediaStreamOptions. + * + * If a fake stream was requested, we force the use of the default backend. + */ + if (fake) { + // Fake stream from default backend. + gUMRunnable = new GetUserMediaRunnable( + audio, video, onSuccess.forget(), onError.forget(), listeners, + windowID, new MediaEngineDefault() + ); + } else if (audiodevice || videodevice) { + // Stream from provided device. + gUMRunnable = new GetUserMediaRunnable( + audio, video, picture, onSuccess.forget(), onError.forget(), listeners, + windowID, + static_cast(audiodevice.get()), + static_cast(videodevice.get()) + ); + } else { + // Stream from default device from WebRTC backend. + gUMRunnable = new GetUserMediaRunnable( + audio, video, picture, onSuccess.forget(), onError.forget(), listeners, + windowID + ); + } } + #ifdef ANDROID if (picture) { // ShowFilePickerForMimeType() must run on the Main Thread! (on Android) @@ -811,11 +837,7 @@ MediaManager::GetUserMedia(bool aPrivileged, nsPIDOMWindow* aWindow, #else // XXX No full support for picture in Desktop yet (needs proper UI) if (aPrivileged || fake) { - if (!mMediaThread) { - nsresult rv = NS_NewThread(getter_AddRefs(mMediaThread)); - NS_ENSURE_SUCCESS(rv, rv); - LOG(("New Media thread for gum")); - } + (void) MediaManager::GetThread(); mMediaThread->Dispatch(gUMRunnable, NS_DISPATCH_NORMAL); } else { // Ask for user permission, and dispatch runnable (or not) when a response @@ -836,7 +858,10 @@ MediaManager::GetUserMedia(bool aPrivileged, nsPIDOMWindow* aWindow, NS_ConvertUTF8toUTF16 callID(buffer); // Store the current callback. - mActiveCallbacks.Put(callID, gUMRunnable); + { + MutexAutoLock lock(mMutex); + mActiveCallbacks.Put(callID, gUMRunnable); + } // Construct JSON structure with both the windowID and the callID. nsAutoString data; @@ -912,21 +937,24 @@ MediaManager::OnNavigation(uint64_t aWindowID) { // Invalidate this window. The runnables check this value before making // a call to content. - StreamListeners* listeners = mActiveWindows.Get(aWindowID); - if (!listeners) { - return; - } + { + MutexAutoLock lock(mMutex); + StreamListeners* listeners = mActiveWindows.Get(aWindowID); + if (!listeners) { + return; + } - uint32_t length = listeners->Length(); - for (uint32_t i = 0; i < length; i++) { - nsRefPtr listener = - listeners->ElementAt(i); - listener->Invalidate(); - listener = nullptr; - } - listeners->Clear(); + uint32_t length = listeners->Length(); + for (uint32_t i = 0; i < length; i++) { + nsRefPtr listener = + listeners->ElementAt(i); + listener->Invalidate(); + listener = nullptr; + } + listeners->Clear(); - mActiveWindows.Remove(aWindowID); + mActiveWindows.Remove(aWindowID); + } } nsresult @@ -942,9 +970,12 @@ MediaManager::Observe(nsISupports* aSubject, const char* aTopic, obs->RemoveObserver(this, "getUserMedia:response:deny"); // Close off any remaining active windows. - mActiveWindows.Clear(); - mActiveCallbacks.Clear(); - sSingleton = nullptr; + { + MutexAutoLock lock(mMutex); + mActiveWindows.Clear(); + mActiveCallbacks.Clear(); + sSingleton = nullptr; + } return NS_OK; } @@ -952,18 +983,15 @@ MediaManager::Observe(nsISupports* aSubject, const char* aTopic, if (!strcmp(aTopic, "getUserMedia:response:allow")) { nsString key(aData); nsRefPtr runnable; - if (!mActiveCallbacks.Get(key, getter_AddRefs(runnable))) { - return NS_OK; + { + MutexAutoLock lock(mMutex); + if (!mActiveCallbacks.Get(key, getter_AddRefs(runnable))) { + return NS_OK; + } } // Reuse the same thread to save memory. - if (!mMediaThread) { - LOG(("New Media thread for gum on allow")); - nsresult rv = NS_NewThread(getter_AddRefs(mMediaThread)); - NS_ENSURE_SUCCESS(rv, rv); - } else { - LOG(("Reused Media thread for gum on allow")); - } + (void) MediaManager::GetThread(); if (aSubject) { // A particular device was chosen by the user. @@ -985,20 +1013,25 @@ MediaManager::Observe(nsISupports* aSubject, const char* aTopic, } mMediaThread->Dispatch(runnable, NS_DISPATCH_NORMAL); - mActiveCallbacks.Remove(key); + { + MutexAutoLock lock(mMutex); + mActiveCallbacks.Remove(key); + } return NS_OK; } if (!strcmp(aTopic, "getUserMedia:response:deny")) { nsString key(aData); nsRefPtr runnable; - if (mActiveCallbacks.Get(key, getter_AddRefs(runnable))) { - GetUserMediaRunnable* gUMRunnable = + { + MutexAutoLock lock(mMutex); + if (mActiveCallbacks.Get(key, getter_AddRefs(runnable))) { + GetUserMediaRunnable* gUMRunnable = static_cast(runnable.get()); - gUMRunnable->Denied(); - mActiveCallbacks.Remove(key); + gUMRunnable->Denied(); + mActiveCallbacks.Remove(key); + } } - return NS_OK; } diff --git a/dom/media/MediaManager.h b/dom/media/MediaManager.h index 0c6d57b88e57..4cde1eb276d0 100644 --- a/dom/media/MediaManager.h +++ b/dom/media/MediaManager.h @@ -65,6 +65,120 @@ class GetUserMediaNotificationEvent: public nsRunnable GetUserMediaStatus mStatus; }; +typedef enum { + MEDIA_START, + MEDIA_STOP, + MEDIA_RELEASE +} MediaOperation; + +// Generic class for running long media operations off the main thread, and +// then (because nsDOMMediaStreams aren't threadsafe), re-sends itseld to +// MainThread to release mStream. This is part of the reason we use an +// operation type - we can change it to repost the runnable to MainThread +// to do operations with the nsDOMMediaStreams, while we can't assign or +// copy a nsRefPtr to a nsDOMMediaStream +class MediaOperationRunnable : public nsRunnable +{ +public: + MediaOperationRunnable(MediaOperation aType, + nsDOMMediaStream* aStream, + MediaEngineSource* aAudioSource, + MediaEngineSource* aVideoSource) + : mType(aType) + , mAudioSource(aAudioSource) + , mVideoSource(aVideoSource) + , mStream(aStream) + {} + + MediaOperationRunnable(MediaOperation aType, + SourceMediaStream* aStream, + MediaEngineSource* aAudioSource, + MediaEngineSource* aVideoSource) + : mType(aType) + , mAudioSource(aAudioSource) + , mVideoSource(aVideoSource) + , mStream(nullptr) + , mSourceStream(aStream) + {} + + NS_IMETHOD + Run() + { + // No locking between these is required as all the callbacks (other + // than MEDIA_RELEASE) for the same MediaStream will occur on the same + // thread. + if (mStream) { + mSourceStream = mStream->GetStream()->AsSourceStream(); + } + switch (mType) { + case MEDIA_START: + { + NS_ASSERTION(!NS_IsMainThread(), "Never call on main thread"); + nsresult rv; + + mSourceStream->SetPullEnabled(true); + + if (mAudioSource) { + rv = mAudioSource->Start(mSourceStream, kAudioTrack); + if (NS_FAILED(rv)) { + MM_LOG(("Starting audio failed, rv=%d",rv)); + } + } + if (mVideoSource) { + rv = mVideoSource->Start(mSourceStream, kVideoTrack); + if (NS_FAILED(rv)) { + MM_LOG(("Starting video failed, rv=%d",rv)); + } + } + + MM_LOG(("started all sources")); + nsCOMPtr event = + new GetUserMediaNotificationEvent(GetUserMediaNotificationEvent::STARTING); + + NS_DispatchToMainThread(event, NS_DISPATCH_NORMAL); + } + break; + + case MEDIA_STOP: + { + NS_ASSERTION(!NS_IsMainThread(), "Never call on main thread"); + if (mAudioSource) { + mAudioSource->Stop(); + mAudioSource->Deallocate(); + } + if (mVideoSource) { + mVideoSource->Stop(); + mVideoSource->Deallocate(); + } + // Do this after stopping all tracks with EndTrack() + mSourceStream->Finish(); + + nsCOMPtr event = + new GetUserMediaNotificationEvent(GetUserMediaNotificationEvent::STOPPING); + + NS_DispatchToMainThread(event, NS_DISPATCH_NORMAL); + } + break; + case MEDIA_RELEASE: + // We go to MainThread to die + break; + } + if (mType != MEDIA_RELEASE) { + // nsDOMMediaStreams aren't thread-safe... sigh. + mType = MEDIA_RELEASE; + NS_DispatchToMainThread(this); + } + return NS_OK; + } + +private: + MediaOperation mType; + nsRefPtr mAudioSource; + nsRefPtr mVideoSource; + nsCOMPtr mStream; + SourceMediaStream *mSourceStream; +}; + /** * This class is an implementation of MediaStreamListener. This is used * to Start() and Stop() the underlying MediaEngineSource when MediaStreams @@ -73,10 +187,12 @@ class GetUserMediaNotificationEvent: public nsRunnable class GetUserMediaCallbackMediaStreamListener : public MediaStreamListener { public: - GetUserMediaCallbackMediaStreamListener(nsDOMMediaStream* aStream, + GetUserMediaCallbackMediaStreamListener(nsIThread *aThread, + nsDOMMediaStream* aStream, MediaEngineSource* aAudioSource, MediaEngineSource* aVideoSource) - : mAudioSource(aAudioSource) + : mMediaThread(aThread) + , mAudioSource(aAudioSource) , mVideoSource(aVideoSource) , mStream(aStream) , mValid(true) {} @@ -84,60 +200,17 @@ public: void Invalidate() { - if (!mValid) { - return; - } + nsRefPtr runnable; - mValid = false; - if (mAudioSource) { - mAudioSource->Stop(); - mAudioSource->Deallocate(); - } - if (mVideoSource) { - mVideoSource->Stop(); - mVideoSource->Deallocate(); - } - // Do this after stopping all tracks with EndTrack() - mStream->GetStream()->AsSourceStream()->Finish(); + // We can't take a chance on blocking here, so proxy this to another + // thread. + // XXX FIX! I'm cheating and passing a raw pointer to the sourcestream + // which is valid as long as the mStream pointer here is. Need a better solution. + runnable = new MediaOperationRunnable(MEDIA_STOP, + mStream->GetStream()->AsSourceStream(), + mAudioSource, mVideoSource); + mMediaThread->Dispatch(runnable, NS_DISPATCH_NORMAL); - nsCOMPtr event = - new GetUserMediaNotificationEvent(GetUserMediaNotificationEvent::STOPPING); - - NS_DispatchToMainThread(event, NS_DISPATCH_NORMAL); - } - - void - NotifyConsumptionChanged(MediaStreamGraph* aGraph, Consumption aConsuming) - { - if (aConsuming == CONSUMED) { - nsresult rv; - - SourceMediaStream* stream = mStream->GetStream()->AsSourceStream(); - stream->SetPullEnabled(true); - - if (mAudioSource) { - rv = mAudioSource->Start(stream, kAudioTrack); - if (NS_FAILED(rv)) { - MM_LOG(("Starting audio failed, rv=%d",rv)); - } - } - if (mVideoSource) { - rv = mVideoSource->Start(stream, kVideoTrack); - if (NS_FAILED(rv)) { - MM_LOG(("Starting video failed, rv=%d",rv)); - } - } - - MM_LOG(("started all sources")); - nsCOMPtr event = - new GetUserMediaNotificationEvent(GetUserMediaNotificationEvent::STARTING); - - NS_DispatchToMainThread(event, NS_DISPATCH_NORMAL); - return; - } - - // NOT_CONSUMED - Invalidate(); return; } @@ -156,6 +229,7 @@ public: } private: + nsCOMPtr mMediaThread; nsRefPtr mAudioSource; nsRefPtr mVideoSource; nsCOMPtr mStream; @@ -204,6 +278,17 @@ public: } return sSingleton; } + static Mutex& GetMutex() { + return Get()->mMutex; + } + static nsIThread* GetThread() { + MutexAutoLock lock(Get()->mMutex); // only need to call Get() once + if (!sSingleton->mMediaThread) { + NS_NewThread(getter_AddRefs(sSingleton->mMediaThread)); + MM_LOG(("New Media thread for gum")); + } + return sSingleton->mMediaThread; + } NS_DECL_ISUPPORTS NS_DECL_NSIOBSERVER @@ -223,17 +308,19 @@ public: private: // Make private because we want only one instance of this class MediaManager() - : mBackend(nullptr) + : mMutex("mozilla::MediaManager") + , mBackend(nullptr) , mMediaThread(nullptr) { mActiveWindows.Init(); mActiveCallbacks.Init(); }; - MediaManager(MediaManager const&) {}; ~MediaManager() { delete mBackend; }; + Mutex mMutex; + // protected with mMutex: MediaEngine* mBackend; nsCOMPtr mMediaThread; WindowTable mActiveWindows;