Bug 1592289 - Implement mute/unmute events for media element capture. r=padenot

Differential Revision: https://phabricator.services.mozilla.com/D52818 --HG-- extra : moz-landing-system : lando
2019-11-20 15:24:34 +00:00 · 2019-11-20 15:24:34 +00:00 · df720f22d1
--- a/dom/html/HTMLMediaElement.cpp
+++ b/dom/html/HTMLMediaElement.cpp
@ -778,30 +778,46 @@ class HTMLMediaElement::MediaStreamRenderer

 class HTMLMediaElement::MediaElementTrackSource
    : public MediaStreamTrackSource,
-      public MediaStreamTrackSource::Sink {
+      public MediaStreamTrackSource::Sink,
+      public MediaStreamTrackConsumer {
 public:
  NS_DECL_ISUPPORTS_INHERITED
  NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(MediaElementTrackSource,
                                           MediaStreamTrackSource)

  /* MediaDecoder track source */
-  MediaElementTrackSource(ProcessedMediaTrack* aTrack, nsIPrincipal* aPrincipal)
-      : MediaStreamTrackSource(aPrincipal, nsString()), mTrack(aTrack) {
+  MediaElementTrackSource(nsISerialEventTarget* aMainThreadEventTarget,
+                          ProcessedMediaTrack* aTrack, nsIPrincipal* aPrincipal,
+                          OutputMuteState aMuteState)
+      : MediaStreamTrackSource(aPrincipal, nsString()),
+        mMainThreadEventTarget(aMainThreadEventTarget),
+        mTrack(aTrack),
+        mIntendedElementMuteState(aMuteState),
+        mElementMuteState(aMuteState) {
    MOZ_ASSERT(mTrack);
  }

  /* MediaStream track source */
-  MediaElementTrackSource(MediaStreamTrackSource* aCapturedTrackSource,
-                          ProcessedMediaTrack* aTrack, MediaInputPort* aPort)
+  MediaElementTrackSource(nsISerialEventTarget* aMainThreadEventTarget,
+                          MediaStreamTrack* aCapturedTrack,
+                          MediaStreamTrackSource* aCapturedTrackSource,
+                          ProcessedMediaTrack* aTrack, MediaInputPort* aPort,
+                          OutputMuteState aMuteState)
      : MediaStreamTrackSource(aCapturedTrackSource->GetPrincipal(),
                               nsString()),
+        mMainThreadEventTarget(aMainThreadEventTarget),
+        mCapturedTrack(aCapturedTrack),
        mCapturedTrackSource(aCapturedTrackSource),
        mTrack(aTrack),
-        mPort(aPort) {
+        mPort(aPort),
+        mIntendedElementMuteState(aMuteState),
+        mElementMuteState(aMuteState) {
    MOZ_ASSERT(mTrack);
+    MOZ_ASSERT(mCapturedTrack);
    MOZ_ASSERT(mCapturedTrackSource);
    MOZ_ASSERT(mPort);

+    mCapturedTrack->AddConsumer(this);
    mCapturedTrackSource->RegisterSink(this);
  }

@ -818,7 +834,24 @@ class HTMLMediaElement::MediaElementTrackSource
    MediaStreamTrackSource::PrincipalChanged();
  }

+  void SetMutedByElement(OutputMuteState aMuteState) {
+    if (mIntendedElementMuteState == aMuteState) {
+      return;
+    }
+    mIntendedElementMuteState = aMuteState;
+    mMainThreadEventTarget->Dispatch(NS_NewRunnableFunction(
+        "MediaElementTrackSource::SetMutedByElement",
+        [self = RefPtr<MediaElementTrackSource>(this), this, aMuteState] {
+          mElementMuteState = aMuteState;
+          MediaStreamTrackSource::MutedChanged(Muted());
+        }));
+  }
+
  void Destroy() override {
+    if (mCapturedTrack) {
+      mCapturedTrack->RemoveConsumer(this);
+      mCapturedTrack = nullptr;
+    }
    if (mCapturedTrackSource) {
      mCapturedTrackSource->UnregisterSink(this);
      mCapturedTrackSource = nullptr;
@ -867,7 +900,7 @@ class HTMLMediaElement::MediaElementTrackSource
  }

  void MutedChanged(bool aNewState) override {
-    MediaStreamTrackSource::MutedChanged(aNewState);
+    MediaStreamTrackSource::MutedChanged(Muted());
  }

  void OverrideEnded() override {
@ -875,14 +908,31 @@ class HTMLMediaElement::MediaElementTrackSource
    MediaStreamTrackSource::OverrideEnded();
  }

+  void NotifyEnabledChanged(MediaStreamTrack* aTrack, bool aEnabled) override {
+    MediaStreamTrackSource::MutedChanged(Muted());
+  }
+
+  bool Muted() const {
+    return mElementMuteState == OutputMuteState::Muted ||
+           (mCapturedTrack &&
+            (mCapturedTrack->Muted() || !mCapturedTrack->Enabled()));
+  }
+
  ProcessedMediaTrack* Track() const { return mTrack; }

 private:
  virtual ~MediaElementTrackSource() { Destroy(); };

+  const RefPtr<nsISerialEventTarget> mMainThreadEventTarget;
+  RefPtr<MediaStreamTrack> mCapturedTrack;
  RefPtr<MediaStreamTrackSource> mCapturedTrackSource;
  const RefPtr<ProcessedMediaTrack> mTrack;
  RefPtr<MediaInputPort> mPort;
+  // The mute state as intended by the media element.
+  OutputMuteState mIntendedElementMuteState;
+  // The mute state as applied to this track source. It is applied async, so
+  // needs to be tracked separately from the intended state.
+  OutputMuteState mElementMuteState;
 };

 HTMLMediaElement::OutputMediaStream::OutputMediaStream(
@ -923,10 +973,12 @@ NS_IMPL_CYCLE_COLLECTION_CLASS(HTMLMediaElement::MediaElementTrackSource)
 NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN_INHERITED(
    HTMLMediaElement::MediaElementTrackSource, MediaStreamTrackSource)
  tmp->Destroy();
+  NS_IMPL_CYCLE_COLLECTION_UNLINK(mCapturedTrack)
  NS_IMPL_CYCLE_COLLECTION_UNLINK(mCapturedTrackSource)
 NS_IMPL_CYCLE_COLLECTION_UNLINK_END
 NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN_INHERITED(
    HTMLMediaElement::MediaElementTrackSource, MediaStreamTrackSource)
+  NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mCapturedTrack)
  NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mCapturedTrackSource)
 NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END

@ -3304,6 +3356,17 @@ void HTMLMediaElement::SetCapturedOutputStreamsEnabled(bool aEnabled) {
  }
 }

+HTMLMediaElement::OutputMuteState HTMLMediaElement::OutputTracksMuted() {
+  return mPaused || mReadyState <= HAVE_CURRENT_DATA ? OutputMuteState::Muted
+                                                     : OutputMuteState::Unmuted;
+}
+
+void HTMLMediaElement::UpdateOutputTracksMuting() {
+  for (auto& entry : mOutputTrackSources) {
+    entry.GetData()->SetMutedByElement(OutputTracksMuted());
+  }
+}
+
 void HTMLMediaElement::AddOutputTrackSourceToOutputStream(
    MediaElementTrackSource* aSource, OutputMediaStream& aOutputStream,
    AddTrackMode aMode) {
@ -3322,11 +3385,13 @@ void HTMLMediaElement::AddOutputTrackSourceToOutputStream(

  RefPtr<MediaStreamTrack> domTrack;
  if (aSource->Track()->mType == MediaSegment::AUDIO) {
-    domTrack = new AudioStreamTrack(aOutputStream.mStream->GetParentObject(),
-                                    aSource->Track(), aSource);
+    domTrack = new AudioStreamTrack(
+        aOutputStream.mStream->GetParentObject(), aSource->Track(), aSource,
+        MediaStreamTrackState::Live, aSource->Muted());
  } else {
-    domTrack = new VideoStreamTrack(aOutputStream.mStream->GetParentObject(),
-                                    aSource->Track(), aSource);
+    domTrack = new VideoStreamTrack(
+        aOutputStream.mStream->GetParentObject(), aSource->Track(), aSource,
+        MediaStreamTrackState::Live, aSource->Muted());
  }

  aOutputStream.mLiveTracks.AppendElement(domTrack);
@ -3504,7 +3569,8 @@ void HTMLMediaElement::UpdateOutputTrackSources() {
      if (!principal || IsCORSSameOrigin()) {
        principal = NodePrincipal();
      }
-      source = MakeAndAddRef<MediaElementTrackSource>(track, principal);
+      source = MakeAndAddRef<MediaElementTrackSource>(
+          mMainThreadEventTarget, track, principal, OutputTracksMuted());
      mDecoder->AddOutputTrack(track);
    } else if (mSrcStream) {
      MediaStreamTrack* inputTrack;
@ -3524,8 +3590,9 @@ void HTMLMediaElement::UpdateOutputTrackSources() {

      track = inputTrack->Graph()->CreateForwardedInputTrack(type);
      RefPtr<MediaInputPort> port = inputTrack->ForwardTrackContentsTo(track);
-      source = MakeAndAddRef<MediaElementTrackSource>(&inputTrack->GetSource(),
-                                                      track, port);
+      source = MakeAndAddRef<MediaElementTrackSource>(
+          mMainThreadEventTarget, inputTrack, &inputTrack->GetSource(), track,
+          port, OutputTracksMuted());

      // Track is muted initially, so we don't leak data if it's added while
      // paused and an MTG iteration passes before the mute comes into effect.
@ -3956,6 +4023,8 @@ void HTMLMediaElement::Init() {
  DecoderDoctorLogger::LogConstruction(this);

  mWatchManager.Watch(mPaused, &HTMLMediaElement::UpdateWakeLock);
+  mWatchManager.Watch(mPaused, &HTMLMediaElement::UpdateOutputTracksMuting);
+  mWatchManager.Watch(mReadyState, &HTMLMediaElement::UpdateOutputTracksMuting);

  mWatchManager.Watch(mTracksCaptured,
                      &HTMLMediaElement::UpdateOutputTrackSources);
--- a/dom/html/HTMLMediaElement.h
+++ b/dom/html/HTMLMediaElement.h
@ -866,6 +866,19 @@ class HTMLMediaElement : public nsGenericHTMLElement,
   */
  void SetCapturedOutputStreamsEnabled(bool aEnabled);

+  /**
+   * Returns true if output tracks should be muted, based on the state of this
+   * media element.
+   */
+  enum class OutputMuteState { Muted, Unmuted };
+  OutputMuteState OutputTracksMuted();
+
+  /**
+   * Sets the muted state of all output track sources. They are muted when we're
+   * paused and unmuted otherwise.
+   */
+  void UpdateOutputTracksMuting();
+
  /**
   * Create a new MediaStreamTrack for the TrackSource corresponding to aTrack
   * and add it to the DOMMediaStream in aOutputStream. This automatically sets
--- a/dom/media/AudioStreamTrack.cpp
+++ b/dom/media/AudioStreamTrack.cpp
@ -42,7 +42,7 @@ void AudioStreamTrack::GetLabel(nsAString& aLabel, CallerType aCallerType) {

 already_AddRefed<MediaStreamTrack> AudioStreamTrack::CloneInternal() {
  return do_AddRef(new AudioStreamTrack(mWindow, mInputTrack, mSource,
-                                        ReadyState(), mConstraints));
+                                        ReadyState(), Muted(), mConstraints));
 }

 }  // namespace dom
--- a/dom/media/AudioStreamTrack.h
+++ b/dom/media/AudioStreamTrack.h
@ -18,8 +18,9 @@ class AudioStreamTrack : public MediaStreamTrack {
      nsPIDOMWindowInner* aWindow, mozilla::MediaTrack* aInputTrack,
      MediaStreamTrackSource* aSource,
      MediaStreamTrackState aReadyState = MediaStreamTrackState::Live,
+      bool aMuted = false,
      const MediaTrackConstraints& aConstraints = MediaTrackConstraints())
-      : MediaStreamTrack(aWindow, aInputTrack, aSource, aReadyState,
+      : MediaStreamTrack(aWindow, aInputTrack, aSource, aReadyState, aMuted,
                         aConstraints) {}

  AudioStreamTrack* AsAudioStreamTrack() override { return this; }
--- a/dom/media/MediaManager.cpp
+++ b/dom/media/MediaManager.cpp
@ -1211,7 +1211,7 @@ class GetUserMediaStreamRunnable : public Runnable {
        MOZ_ASSERT(IsOn(mConstraints.mAudio));
        RefPtr<MediaStreamTrack> domTrack = new dom::AudioStreamTrack(
            window, track, audioTrackSource, dom::MediaStreamTrackState::Live,
-            GetInvariant(mConstraints.mAudio));
+            false, GetInvariant(mConstraints.mAudio));
        domStream->AddTrackInternal(domTrack);
      }
    }
@ -1225,7 +1225,7 @@ class GetUserMediaStreamRunnable : public Runnable {
      MOZ_ASSERT(IsOn(mConstraints.mVideo));
      RefPtr<MediaStreamTrack> domTrack = new dom::VideoStreamTrack(
          window, track, videoTrackSource, dom::MediaStreamTrackState::Live,
-          GetInvariant(mConstraints.mVideo));
+          false, GetInvariant(mConstraints.mVideo));
      domStream->AddTrackInternal(domTrack);
      switch (mVideoDevice->GetMediaSource()) {
        case MediaSourceEnum::Browser:
--- a/dom/media/MediaStreamTrack.cpp
+++ b/dom/media/MediaStreamTrack.cpp
@ -186,6 +186,7 @@ MediaStreamTrack::MediaStreamTrack(nsPIDOMWindowInner* aWindow,
                                   mozilla::MediaTrack* aInputTrack,
                                   MediaStreamTrackSource* aSource,
                                   MediaStreamTrackState aReadyState,
+                                   bool aMuted,
                                   const MediaTrackConstraints& aConstraints)
    : mWindow(aWindow),
      mInputTrack(aInputTrack),
@ -194,7 +195,7 @@ MediaStreamTrack::MediaStreamTrack(nsPIDOMWindowInner* aWindow,
      mPrincipal(aSource->GetPrincipal()),
      mReadyState(aReadyState),
      mEnabled(true),
-      mMuted(false),
+      mMuted(aMuted),
      mConstraints(aConstraints) {
  if (!Ended()) {
    GetSource().RegisterSink(mSink.get());
@ -297,7 +298,7 @@ void MediaStreamTrack::SetEnabled(bool aEnabled) {

  mTrack->SetEnabled(mEnabled ? DisabledTrackMode::ENABLED
                              : DisabledTrackMode::SILENCE_BLACK);
-  GetSource().SinkEnabledStateChanged();
+  NotifyEnabledChanged();
 }

 void MediaStreamTrack::Stop() {
@ -478,6 +479,20 @@ void MediaStreamTrack::NotifyEnded() {
  }
 }

+void MediaStreamTrack::NotifyEnabledChanged() {
+  GetSource().SinkEnabledStateChanged();
+
+  auto consumers(mConsumers);
+  for (const auto& consumer : consumers) {
+    if (consumer) {
+      consumer->NotifyEnabledChanged(this, Enabled());
+    } else {
+      MOZ_ASSERT_UNREACHABLE("A consumer was not explicitly removed");
+      mConsumers.RemoveElement(consumer);
+    }
+  }
+}
+
 bool MediaStreamTrack::AddPrincipalChangeObserver(
    PrincipalChangeObserver<MediaStreamTrack>* aObserver) {
  return mPrincipalChangeObservers.AppendElement(aObserver) != nullptr;
--- a/dom/media/MediaStreamTrack.h
+++ b/dom/media/MediaStreamTrack.h
@ -359,6 +359,11 @@ class MediaStreamTrackConsumer
   * including MediaStreamTrack::Stop().
   */
  virtual void NotifyEnded(MediaStreamTrack* aTrack){};
+
+  /**
+   * Called when the track's enabled state changes.
+   */
+  virtual void NotifyEnabledChanged(MediaStreamTrack* aTrack, bool aEnabled){};
 };

 // clang-format off
@ -414,6 +419,7 @@ class MediaStreamTrack : public DOMEventTargetHelper,
      nsPIDOMWindowInner* aWindow, mozilla::MediaTrack* aInputTrack,
      MediaStreamTrackSource* aSource,
      MediaStreamTrackState aReadyState = MediaStreamTrackState::Live,
+      bool aMuted = false,
      const MediaTrackConstraints& aConstraints = MediaTrackConstraints());

  NS_DECL_ISUPPORTS_INHERITED
@ -576,6 +582,12 @@ class MediaStreamTrack : public DOMEventTargetHelper,
   */
  void NotifyEnded();

+  /**
+   * Called when this track's enabled state has changed.
+   * Notifies all MediaStreamTrackConsumers.
+   */
+  void NotifyEnabledChanged();
+
  /**
   * Called when mSource's principal has changed.
   */
--- a/dom/media/VideoStreamTrack.cpp
+++ b/dom/media/VideoStreamTrack.cpp
@ -18,8 +18,9 @@ VideoStreamTrack::VideoStreamTrack(nsPIDOMWindowInner* aWindow,
                                   mozilla::MediaTrack* aInputTrack,
                                   MediaStreamTrackSource* aSource,
                                   MediaStreamTrackState aReadyState,
+                                   bool aMuted,
                                   const MediaTrackConstraints& aConstraints)
-    : MediaStreamTrack(aWindow, aInputTrack, aSource, aReadyState,
+    : MediaStreamTrack(aWindow, aInputTrack, aSource, aReadyState, aMuted,
                       aConstraints) {}

 void VideoStreamTrack::Destroy() {
@ -82,7 +83,7 @@ void VideoStreamTrack::GetLabel(nsAString& aLabel, CallerType aCallerType) {

 already_AddRefed<MediaStreamTrack> VideoStreamTrack::CloneInternal() {
  return do_AddRef(new VideoStreamTrack(mWindow, mInputTrack, mSource,
-                                        ReadyState(), mConstraints));
+                                        ReadyState(), Muted(), mConstraints));
 }

 }  // namespace dom
--- a/dom/media/VideoStreamTrack.h
+++ b/dom/media/VideoStreamTrack.h
@ -22,6 +22,7 @@ class VideoStreamTrack : public MediaStreamTrack {
      nsPIDOMWindowInner* aWindow, mozilla::MediaTrack* aInputTrack,
      MediaStreamTrackSource* aSource,
      MediaStreamTrackState aState = MediaStreamTrackState::Live,
+      bool aMuted = false,
      const MediaTrackConstraints& aConstraints = MediaTrackConstraints());

  void Destroy() override;