From 2ab300ac7ba670ec5c567b379ade029133eb5ac2 Mon Sep 17 00:00:00 2001 From: Paul Adenot Date: Fri, 24 Jul 2015 14:28:17 +0200 Subject: [PATCH] Bug 1156472 - Part 6 - Connect HTMLMediaElement and AudioContext to the capture stream when capturing is needed. r=roc --- dom/html/HTMLMediaElement.cpp | 45 ++++++++++++++++----- dom/html/HTMLMediaElement.h | 6 +++ dom/media/webaudio/AudioDestinationNode.cpp | 29 ++++++++----- dom/media/webaudio/AudioDestinationNode.h | 2 + 4 files changed, 64 insertions(+), 18 deletions(-) diff --git a/dom/html/HTMLMediaElement.cpp b/dom/html/HTMLMediaElement.cpp index bfa6d938a214..790103de8c1d 100644 --- a/dom/html/HTMLMediaElement.cpp +++ b/dom/html/HTMLMediaElement.cpp @@ -2030,6 +2030,7 @@ HTMLMediaElement::HTMLMediaElement(already_AddRefed& aNo mAllowCasting(false), mIsCasting(false), mAudioCaptured(false), + mAudioCapturedByWindow(false), mPlayingBeforeSeek(false), mPlayingThroughTheAudioChannelBeforeSeek(false), mPausedForInactiveDocumentOrChannel(false), @@ -2097,6 +2098,11 @@ HTMLMediaElement::~HTMLMediaElement() EndSrcMediaStreamPlayback(); } + if (mCaptureStreamPort) { + mCaptureStreamPort->Destroy(); + mCaptureStreamPort = nullptr; + } + NS_ASSERTION(MediaElementTableCount(this, mLoadingSrc) == 0, "Destroyed media element should no longer be in element table"); @@ -4475,8 +4481,7 @@ void HTMLMediaElement::UpdateAudioChannelPlayingState() (!mPaused && (HasAttr(kNameSpaceID_None, nsGkAtoms::loop) || (mReadyState >= nsIDOMHTMLMediaElement::HAVE_CURRENT_DATA && - !IsPlaybackEnded() && - (!mSrcStream || HasAudio())) || + !IsPlaybackEnded()) || mPlayingThroughTheAudioChannelBeforeSeek)); if (playingThroughTheAudioChannel != mPlayingThroughTheAudioChannel) { mPlayingThroughTheAudioChannel = playingThroughTheAudioChannel; @@ -4504,9 +4509,9 @@ void HTMLMediaElement::UpdateAudioChannelPlayingState() void HTMLMediaElement::NotifyAudioChannelAgent(bool aPlaying) { - // Immediately check if this should go to the MSG instead of the normal - // media playback route. - WindowAudioCaptureChanged(); + // Immediately check if this should go to the MSG instead of the normal + // media playback route. + WindowAudioCaptureChanged(); // This is needed to pass nsContentUtils::IsCallerChrome(). // AudioChannel API should not called from content but it can happen that @@ -4681,11 +4686,33 @@ HTMLMediaElement::GetTopLevelPrincipal() NS_IMETHODIMP HTMLMediaElement::WindowAudioCaptureChanged() { - MOZ_ASSERT(mAudioChannelAgent); - DebugOnly captured = OwnerDoc()->GetInnerWindow()->GetAudioCaptured(); + MOZ_ASSERT(mAudioChannelAgent); - // Something is going to happen here!! - return NS_OK; + if (!OwnerDoc()->GetInnerWindow()) { + return NS_OK; + } + bool captured = OwnerDoc()->GetInnerWindow()->GetAudioCaptured(); + + if (captured != mAudioCapturedByWindow) { + if (captured) { + mAudioCapturedByWindow = true; + nsCOMPtr window = + do_QueryInterface(OwnerDoc()->GetParentObject()); + uint64_t id = window->WindowID(); + MediaStreamGraph* msg = MediaStreamGraph::GetInstance(); + + if (!mPlaybackStream) { + nsRefPtr stream = CaptureStreamInternal(false, msg); + mCaptureStreamPort = msg->ConnectToCaptureStream(id, stream->GetStream()); + } else { + mCaptureStreamPort = msg->ConnectToCaptureStream(id, mPlaybackStream->GetStream()); + } + } else { + // TODO: uncapture + } + } + + return NS_OK; } AudioTrackList* diff --git a/dom/html/HTMLMediaElement.h b/dom/html/HTMLMediaElement.h index 84f4ed0d9cd8..406478fc507c 100644 --- a/dom/html/HTMLMediaElement.h +++ b/dom/html/HTMLMediaElement.h @@ -1074,6 +1074,9 @@ protected: // Holds a reference to a MediaInputPort connecting mSrcStream to mPlaybackStream. nsRefPtr mPlaybackStreamInputPort; + // Holds a reference to the stream connecting this stream to the capture sink. + nsRefPtr mCaptureStreamPort; + // Holds a reference to a stream with mSrcStream as input but intended for // playback. Used so we don't block playback of other video elements // playing the same mSrcStream. @@ -1283,6 +1286,9 @@ protected: // True if the sound is being captured. bool mAudioCaptured; + // True if the sound is being captured by the window. + bool mAudioCapturedByWindow; + // If TRUE then the media element was actively playing before the currently // in progress seeking. If FALSE then the media element is either not seeking // or was not actively playing before the current seek. Used to decide whether diff --git a/dom/media/webaudio/AudioDestinationNode.cpp b/dom/media/webaudio/AudioDestinationNode.cpp index 3f5a3650a0df..d37145ef547a 100644 --- a/dom/media/webaudio/AudioDestinationNode.cpp +++ b/dom/media/webaudio/AudioDestinationNode.cpp @@ -313,12 +313,9 @@ AudioDestinationNode::AudioDestinationNode(AudioContext* aContext, bool aIsOffline, AudioChannel aChannel, uint32_t aNumberOfChannels, - uint32_t aLength, - float aSampleRate) - : AudioNode(aContext, - aIsOffline ? aNumberOfChannels : 2, - ChannelCountMode::Explicit, - ChannelInterpretation::Speakers) + uint32_t aLength, float aSampleRate) + : AudioNode(aContext, aIsOffline ? aNumberOfChannels : 2, + ChannelCountMode::Explicit, ChannelInterpretation::Speakers) , mFramesToProduce(aLength) , mAudioChannel(AudioChannel::Normal) , mIsOffline(aIsOffline) @@ -326,6 +323,7 @@ AudioDestinationNode::AudioDestinationNode(AudioContext* aContext, , mExtraCurrentTime(0) , mExtraCurrentTimeSinceLastStartedBlocking(0) , mExtraCurrentTimeUpdatedSinceLastStableState(false) + , mCaptured(false) { bool startWithAudioDriver = true; MediaStreamGraph* graph = aIsOffline ? @@ -510,13 +508,25 @@ AudioDestinationNode::WindowAudioCaptureChanged() { MOZ_ASSERT(mAudioChannelAgent); - if (!mStream) { + if (!mStream || Context()->IsOffline()) { return NS_OK; } - DebugOnly captured = GetOwner()->GetAudioCaptured(); + bool captured = GetOwner()->GetAudioCaptured(); + + if (captured != mCaptured) { + if (captured) { + nsCOMPtr window = Context()->GetParentObject(); + uint64_t id = window->WindowID(); + mCaptureStreamPort = + mStream->Graph()->ConnectToCaptureStream(id, mStream); + } else { + mCaptureStreamPort->Disconnect(); + mCaptureStreamPort->Destroy(); + } + mCaptured = captured; + } - // XXXtodopadenot actually capture return NS_OK; } @@ -699,6 +709,7 @@ AudioDestinationNode::InputMuted(bool aMuted) return; } + WindowAudioCaptureChanged(); WindowVolumeChanged(volume, muted); } diff --git a/dom/media/webaudio/AudioDestinationNode.h b/dom/media/webaudio/AudioDestinationNode.h index db5418a9cb51..724e44d61d3c 100644 --- a/dom/media/webaudio/AudioDestinationNode.h +++ b/dom/media/webaudio/AudioDestinationNode.h @@ -99,6 +99,7 @@ private: uint32_t mFramesToProduce; nsCOMPtr mAudioChannelAgent; + nsRefPtr mCaptureStreamPort; nsRefPtr mOfflineRenderingPromise; @@ -111,6 +112,7 @@ private: double mExtraCurrentTime; double mExtraCurrentTimeSinceLastStartedBlocking; bool mExtraCurrentTimeUpdatedSinceLastStableState; + bool mCaptured; }; } // namespace dom