Bug 1324548 - Add MediaStreamTrackAudioSourceNode. r=pehrsons,baku

MozReview-Commit-ID: IdVqfNigMyu

Differential Revision: https://phabricator.services.mozilla.com/D15958

--HG--
extra : moz-landing-system : lando

dom/locales/en-US/chrome/dom/dom.properties

@@ -105,6 +105,8 @@ MediaDecodeAudioDataNoAudio=The buffer passed to decodeAudioData does not contai
 MediaElementAudioSourceNodeCrossOrigin=The HTMLMediaElement passed to createMediaElementSource has a cross-origin resource, the node will output silence.
 # LOCALIZATION NOTE: Do not translate MediaStream and createMediaStreamSource.
 MediaStreamAudioSourceNodeCrossOrigin=The MediaStream passed to createMediaStreamSource has a cross-origin resource, the node will output silence.
+# LOCALIZATION NOTE : Do not translate MediaStreamTrack and createMediaStreamTrackSource.
+MediaStreamTrackAudioSourceNodeCrossOrigin=The MediaStreamTrack passed to createMediaStreamTrackSource is a cross-origin resource, the node will output silence.
 # LOCALIZATION NOTE: Do not translate HTMLMediaElement and MediaStream.
 MediaElementAudioCaptureOfMediaStreamError=The captured HTMLMediaElement is playing a MediaStream. Applying volume or mute status is not currently supported.
 MediaLoadExhaustedCandidates=All candidate resources failed to load. Media load paused.

dom/media/MediaStreamTrack.cpp

@@ -112,6 +112,10 @@ class MediaStreamTrack::MSGListener : public MediaStreamTrackListener {
     mGraph->DispatchToMainThreadStableState(
         NS_NewRunnableFunction("MediaStreamTrack::MSGListener::mTrackReleaser",
                                [self = RefPtr<MSGListener>(this)]() {}));
+
+    mGraph->Dispatch(NS_NewRunnableFunction(
+        "MediaStreamTrack::MSGListener::DoNotifyEnded",
+        [self = RefPtr<MSGListener>(this)]() { self->DoNotifyEnded(); }));
   }
 
   void DoNotifyEnded() {

dom/media/webaudio/AudioContext.cpp

@@ -31,6 +31,7 @@
 #include "mozilla/dom/HTMLMediaElement.h"
 #include "mozilla/dom/MediaElementAudioSourceNodeBinding.h"
 #include "mozilla/dom/MediaStreamAudioSourceNodeBinding.h"
+#include "mozilla/dom/MediaStreamTrackAudioSourceNodeBinding.h"
 #include "mozilla/dom/OfflineAudioContextBinding.h"
 #include "mozilla/dom/OscillatorNodeBinding.h"
 #include "mozilla/dom/PannerNodeBinding.h"
@@ -62,6 +63,7 @@
 #include "MediaElementAudioSourceNode.h"
 #include "MediaStreamAudioDestinationNode.h"
 #include "MediaStreamAudioSourceNode.h"
+#include "MediaStreamTrackAudioSourceNode.h"
 #include "MediaStreamGraph.h"
 #include "nsContentUtils.h"
 #include "nsIScriptError.h"
@@ -424,6 +426,15 @@ AudioContext::CreateMediaStreamSource(DOMMediaStream& aMediaStream,
   return MediaStreamAudioSourceNode::Create(*this, options, aRv);
 }
 
+already_AddRefed<MediaStreamTrackAudioSourceNode>
+AudioContext::CreateMediaStreamTrackSource(MediaStreamTrack& aMediaStreamTrack,
+                                           ErrorResult& aRv) {
+  MediaStreamTrackAudioSourceOptions options;
+  options.mMediaStreamTrack = aMediaStreamTrack;
+
+  return MediaStreamTrackAudioSourceNode::Create(*this, options, aRv);
+}
+
 already_AddRefed<GainNode> AudioContext::CreateGain(ErrorResult& aRv) {
   return GainNode::Create(*this, GainOptions(), aRv);
 }

dom/media/webaudio/AudioContext.h

@@ -67,6 +67,8 @@ class IIRFilterNode;
 class MediaElementAudioSourceNode;
 class MediaStreamAudioDestinationNode;
 class MediaStreamAudioSourceNode;
+class MediaStreamTrack;
+class MediaStreamTrackAudioSourceNode;
 class OscillatorNode;
 class PannerNode;
 class ScriptProcessorNode;
@@ -246,6 +248,9 @@ class AudioContext final : public DOMEventTargetHelper,
       HTMLMediaElement& aMediaElement, ErrorResult& aRv);
   already_AddRefed<MediaStreamAudioSourceNode> CreateMediaStreamSource(
       DOMMediaStream& aMediaStream, ErrorResult& aRv);
+  already_AddRefed<MediaStreamTrackAudioSourceNode>
+  CreateMediaStreamTrackSource(MediaStreamTrack& aMediaStreamTrack,
+                               ErrorResult& aRv);
 
   already_AddRefed<DelayNode> CreateDelay(double aMaxDelayTime,
                                           ErrorResult& aRv);

dom/media/webaudio/MediaStreamTrackAudioSourceNode.cpp

@@ -0,0 +1,192 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "MediaStreamTrackAudioSourceNode.h"
+#include "mozilla/dom/MediaStreamTrackAudioSourceNodeBinding.h"
+#include "AudioNodeEngine.h"
+#include "AudioNodeExternalInputStream.h"
+#include "AudioStreamTrack.h"
+#include "mozilla/dom/Document.h"
+#include "mozilla/CORSMode.h"
+#include "nsContentUtils.h"
+#include "nsIScriptError.h"
+
+namespace mozilla {
+namespace dom {
+
+NS_IMPL_CYCLE_COLLECTION_CLASS(MediaStreamTrackAudioSourceNode)
+
+NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN(MediaStreamTrackAudioSourceNode)
+  tmp->Destroy();
+  NS_IMPL_CYCLE_COLLECTION_UNLINK(mInputTrack)
+NS_IMPL_CYCLE_COLLECTION_UNLINK_END_INHERITED(AudioNode)
+
+NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN_INHERITED(
+    MediaStreamTrackAudioSourceNode, AudioNode)
+  NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mInputTrack)
+NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END
+
+NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(MediaStreamTrackAudioSourceNode)
+NS_INTERFACE_MAP_END_INHERITING(AudioNode)
+
+NS_IMPL_ADDREF_INHERITED(MediaStreamTrackAudioSourceNode, AudioNode)
+NS_IMPL_RELEASE_INHERITED(MediaStreamTrackAudioSourceNode, AudioNode)
+
+MediaStreamTrackAudioSourceNode::MediaStreamTrackAudioSourceNode(
+    AudioContext* aContext)
+    : AudioNode(aContext, 2, ChannelCountMode::Max,
+                ChannelInterpretation::Speakers),
+      mTrackListener(this) {}
+
+/* static */ already_AddRefed<MediaStreamTrackAudioSourceNode>
+MediaStreamTrackAudioSourceNode::Create(
+    AudioContext& aAudioContext,
+    const MediaStreamTrackAudioSourceOptions& aOptions, ErrorResult& aRv) {
+  if (aAudioContext.IsOffline()) {
+    aRv.Throw(NS_ERROR_DOM_NOT_SUPPORTED_ERR);
+    return nullptr;
+  }
+
+  if (aAudioContext.Graph() != aOptions.mMediaStreamTrack->Graph()) {
+    nsCOMPtr<nsPIDOMWindowInner> pWindow = aAudioContext.GetParentObject();
+    Document* document = pWindow ? pWindow->GetExtantDoc() : nullptr;
+    nsContentUtils::ReportToConsole(nsIScriptError::warningFlag,
+                                    NS_LITERAL_CSTRING("Web Audio"), document,
+                                    nsContentUtils::eDOM_PROPERTIES,
+                                    "MediaStreamAudioSourceNodeDifferentRate");
+    aRv.Throw(NS_ERROR_DOM_NOT_SUPPORTED_ERR);
+    return nullptr;
+  }
+
+  RefPtr<MediaStreamTrackAudioSourceNode> node =
+      new MediaStreamTrackAudioSourceNode(&aAudioContext);
+
+  node->Init(aOptions.mMediaStreamTrack, aRv);
+  if (aRv.Failed()) {
+    return nullptr;
+  }
+
+  return node.forget();
+}
+
+void MediaStreamTrackAudioSourceNode::Init(MediaStreamTrack* aMediaStreamTrack,
+                                           ErrorResult& aRv) {
+  MOZ_ASSERT(aMediaStreamTrack);
+
+  if (!aMediaStreamTrack->AsAudioStreamTrack()) {
+    mTrackListener.NotifyEnded(nullptr);
+    aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR);
+    return;
+  }
+
+  MediaStreamGraph* graph = Context()->Graph();
+
+  AudioNodeEngine* engine = new MediaStreamTrackAudioSourceNodeEngine(this);
+  mStream = AudioNodeExternalInputStream::Create(graph, engine);
+
+  MOZ_ASSERT(mStream);
+
+  mInputTrack = aMediaStreamTrack;
+  ProcessedMediaStream* outputStream =
+      static_cast<ProcessedMediaStream*>(mStream.get());
+  mInputPort = mInputTrack->ForwardTrackContentsTo(outputStream);
+  PrincipalChanged(mInputTrack);  // trigger enabling/disabling of the connector
+  mInputTrack->AddPrincipalChangeObserver(this);
+
+  mInputTrack->AddConsumer(&mTrackListener);
+}
+
+void MediaStreamTrackAudioSourceNode::Destroy() {
+  if (mInputTrack) {
+    mInputTrack->RemovePrincipalChangeObserver(this);
+    mInputTrack->RemoveConsumer(&mTrackListener);
+    mInputTrack = nullptr;
+  }
+  mTrackListener.NotifyEnded(mInputTrack);
+
+  if (mInputPort) {
+    mInputPort->Destroy();
+    mInputPort = nullptr;
+  }
+}
+
+MediaStreamTrackAudioSourceNode::~MediaStreamTrackAudioSourceNode() {
+  Destroy();
+}
+
+/**
+ * Changes the principal. Note that this will be called on the main thread, but
+ * changes will be enacted on the MediaStreamGraph thread. If the principal
+ * change results in the document principal losing access to the stream, then
+ * there needs to be other measures in place to ensure that any media that is
+ * governed by the new stream principal is not available to the MediaStreamGraph
+ * before this change completes. Otherwise, a site could get access to
+ * media that they are not authorized to receive.
+ *
+ * One solution is to block the altered content, call this method, then dispatch
+ * another change request to the MediaStreamGraph thread that allows the content
+ * under the new principal to flow. This might be unnecessary if the principal
+ * change is changing to be the document principal.
+ */
+void MediaStreamTrackAudioSourceNode::PrincipalChanged(
+    MediaStreamTrack* aMediaStreamTrack) {
+  MOZ_ASSERT(aMediaStreamTrack == mInputTrack);
+
+  bool subsumes = false;
+  Document* doc = nullptr;
+  if (nsPIDOMWindowInner* parent = Context()->GetParentObject()) {
+    doc = parent->GetExtantDoc();
+    if (doc) {
+      nsIPrincipal* docPrincipal = doc->NodePrincipal();
+      nsIPrincipal* trackPrincipal = aMediaStreamTrack->GetPrincipal();
+      if (!trackPrincipal ||
+          NS_FAILED(docPrincipal->Subsumes(trackPrincipal, &subsumes))) {
+        subsumes = false;
+      }
+    }
+  }
+  auto stream = static_cast<AudioNodeExternalInputStream*>(mStream.get());
+  bool enabled = subsumes || aMediaStreamTrack->GetCORSMode() != CORS_NONE;
+  stream->SetInt32Parameter(MediaStreamTrackAudioSourceNodeEngine::ENABLE,
+                            enabled);
+  fprintf(stderr, "NOW: %s", enabled ? "enabled" : "disabled");
+
+  if (!enabled && doc) {
+    nsContentUtils::ReportToConsole(
+        nsIScriptError::warningFlag, NS_LITERAL_CSTRING("Web Audio"), doc,
+        nsContentUtils::eDOM_PROPERTIES, CrossOriginErrorString());
+  }
+}
+
+size_t MediaStreamTrackAudioSourceNode::SizeOfExcludingThis(
+    MallocSizeOf aMallocSizeOf) const {
+  size_t amount = AudioNode::SizeOfExcludingThis(aMallocSizeOf);
+  if (mInputPort) {
+    amount += mInputPort->SizeOfIncludingThis(aMallocSizeOf);
+  }
+  return amount;
+}
+
+size_t MediaStreamTrackAudioSourceNode::SizeOfIncludingThis(
+    MallocSizeOf aMallocSizeOf) const {
+  return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
+}
+
+void MediaStreamTrackAudioSourceNode::DestroyMediaStream() {
+  if (mInputPort) {
+    mInputPort->Destroy();
+    mInputPort = nullptr;
+  }
+  AudioNode::DestroyMediaStream();
+}
+
+JSObject* MediaStreamTrackAudioSourceNode::WrapObject(
+    JSContext* aCx, JS::Handle<JSObject*> aGivenProto) {
+  return MediaStreamTrackAudioSourceNode_Binding::Wrap(aCx, this, aGivenProto);
+}
+
+}  // namespace dom
+}  // namespace mozilla

dom/media/webaudio/MediaStreamTrackAudioSourceNode.h

@@ -0,0 +1,115 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef MediaStreamTrackAudioSourceNode_h_
+#define MediaStreamTrackAudioSourceNode_h_
+
+#include "AudioNode.h"
+#include "AudioNodeEngine.h"
+#include "mozilla/WeakPtr.h"
+
+namespace mozilla {
+
+namespace dom {
+
+class AudioContext;
+struct MediaStreamTrackAudioSourceOptions;
+
+class MediaStreamTrackAudioSourceNodeEngine final : public AudioNodeEngine {
+ public:
+  explicit MediaStreamTrackAudioSourceNodeEngine(AudioNode* aNode)
+      : AudioNodeEngine(aNode), mEnabled(false) {}
+
+  bool IsEnabled() const { return mEnabled; }
+  enum Parameters { ENABLE };
+  void SetInt32Parameter(uint32_t aIndex, int32_t aValue) override {
+    switch (aIndex) {
+      case ENABLE:
+        mEnabled = !!aValue;
+        break;
+      default:
+        NS_ERROR("MediaStreamTrackAudioSourceNodeEngine bad parameter index");
+    }
+  }
+
+ private:
+  bool mEnabled;
+};
+
+class MediaStreamTrackAudioSourceNode
+    : public AudioNode,
+      public PrincipalChangeObserver<MediaStreamTrack>,
+      public SupportsWeakPtr<MediaStreamTrackAudioSourceNode> {
+ public:
+  static already_AddRefed<MediaStreamTrackAudioSourceNode> Create(
+      AudioContext& aContext,
+      const MediaStreamTrackAudioSourceOptions& aOptions, ErrorResult& aRv);
+
+  NS_DECL_ISUPPORTS_INHERITED
+  MOZ_DECLARE_WEAKREFERENCE_TYPENAME(MediaStreamTrackAudioSourceNode)
+  NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(MediaStreamTrackAudioSourceNode,
+                                           AudioNode)
+
+  static already_AddRefed<MediaStreamTrackAudioSourceNode> Constructor(
+      const GlobalObject& aGlobal, AudioContext& aAudioContext,
+      const MediaStreamTrackAudioSourceOptions& aOptions, ErrorResult& aRv) {
+    return Create(aAudioContext, aOptions, aRv);
+  }
+
+  JSObject* WrapObject(JSContext* aCx,
+                       JS::Handle<JSObject*> aGivenProto) override;
+
+  void DestroyMediaStream() override;
+
+  uint16_t NumberOfInputs() const override { return 0; }
+
+  const char* NodeType() const override {
+    return "MediaStreamTrackAudioSourceNode";
+  }
+
+  virtual const char* CrossOriginErrorString() const {
+    return "MediaStreamTrackAudioSourceNodeCrossOrigin";
+  }
+
+  size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const override;
+  size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const override;
+
+  // From PrincipalChangeObserver<MediaStreamTrack>.
+  void PrincipalChanged(MediaStreamTrack* aMediaStreamTrack) override;
+
+ protected:
+  explicit MediaStreamTrackAudioSourceNode(AudioContext* aContext);
+  void Init(MediaStreamTrack* aMediaStreamTrack, ErrorResult& aRv);
+  void Destroy();
+  virtual ~MediaStreamTrackAudioSourceNode();
+
+  class TrackListener : public MediaStreamTrackConsumer {
+   public:
+    explicit TrackListener(MediaStreamTrackAudioSourceNode* aNode)
+        : mNode(aNode) {}
+
+    void NotifyEnded(MediaStreamTrack* aTrack) override {
+      if (mNode) {
+        mNode->MarkInactive();
+        mNode->DestroyMediaStream();
+        mNode = nullptr;
+      }
+    }
+
+   private:
+    WeakPtr<MediaStreamTrackAudioSourceNode> mNode;
+  };
+
+ private:
+  RefPtr<MediaInputPort> mInputPort;
+  RefPtr<MediaStreamTrack> mInputTrack;
+  TrackListener mTrackListener;
+};
+
+}  // namespace dom
+}  // namespace mozilla
+
+#endif

dom/media/webaudio/moz.build

@@ -66,6 +66,7 @@ EXPORTS.mozilla.dom += [
     'MediaElementAudioSourceNode.h',
     'MediaStreamAudioDestinationNode.h',
     'MediaStreamAudioSourceNode.h',
+    'MediaStreamTrackAudioSourceNode.h',
     'OscillatorNode.h',
     'PannerNode.h',
     'PeriodicWave.h',
@@ -110,6 +111,7 @@ UNIFIED_SOURCES += [
     'MediaElementAudioSourceNode.cpp',
     'MediaStreamAudioDestinationNode.cpp',
     'MediaStreamAudioSourceNode.cpp',
+    'MediaStreamTrackAudioSourceNode.cpp',
     'OscillatorNode.cpp',
     'PannerNode.cpp',
     'PeriodicWave.cpp',

dom/tests/mochitest/general/test_interfaces.js

@@ -655,6 +655,8 @@ var interfaceNamesInGlobalScope =
     {name: "MediaStreamAudioDestinationNode", insecureContext: true},
 // IMPORTANT: Do not change this list without review from a DOM peer!
     {name: "MediaStreamAudioSourceNode", insecureContext: true},
+// IMPORTANT: Do not change this list without review from a DOM peer!
+    {name: "MediaStreamTrackAudioSourceNode", insecureContext: true},
 // IMPORTANT: Do not change this list without review from a DOM peer!
     {name: "MediaStreamEvent", insecureContext: true},
 // IMPORTANT: Do not change this list without review from a DOM peer!

dom/webidl/AudioContext.webidl

@@ -32,7 +32,8 @@ interface AudioContext : BaseAudioContext {
     [NewObject, Throws]
     MediaStreamAudioSourceNode createMediaStreamSource(MediaStream mediaStream);
 
-    // Bug 1324548: MediaStreamTrackAudioSourceNode createMediaStreamTrackSource (AudioMediaStreamTrack mediaStreamTrack);
+    [NewObject, Throws]
+    MediaStreamTrackAudioSourceNode createMediaStreamTrackSource(MediaStreamTrack mediaStreamTrack);
 
     [NewObject, Throws]
     MediaStreamAudioDestinationNode createMediaStreamDestination();

dom/webidl/MediaStreamTrackAudioSourceNode.webidl

@@ -0,0 +1,24 @@
+/* -*- Mode: IDL; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * The origin of this IDL file is
+ * https://webaudio.github.io/web-audio-api/
+ *
+ * Copyright © 2012 W3C® (MIT, ERCIM, Keio), All Rights Reserved. W3C
+ * liability, trademark and document use rules apply.
+ */
+
+dictionary MediaStreamTrackAudioSourceOptions {
+    required MediaStreamTrack mediaStreamTrack;
+};
+
+[Pref="dom.webaudio.enabled",
+ Constructor(AudioContext context, MediaStreamTrackAudioSourceOptions options)]
+interface MediaStreamTrackAudioSourceNode : AudioNode {
+
+};
+
+// Mozilla extensions
+MediaStreamTrackAudioSourceNode implements AudioNodePassThrough;

dom/webidl/moz.build

@@ -178,6 +178,9 @@ with Files("Media*Source*"):
 with Files("MediaStream*"):
     BUG_COMPONENT = ("Core", "WebRTC")
 
+with Files("MediaStreamTrackAudio*"):
+    BUG_COMPONENT = ("Core", "Web Audio")
+
 with Files("MediaStreamAudio*"):
     BUG_COMPONENT = ("Core", "Web Audio")
 
@@ -657,6 +660,7 @@ WEBIDL_FILES = [
     'MediaStreamAudioSourceNode.webidl',
     'MediaStreamError.webidl',
     'MediaStreamTrack.webidl',
+    'MediaStreamTrackAudioSourceNode.webidl',
     'MediaTrackConstraintSet.webidl',
     'MediaTrackSettings.webidl',
     'MediaTrackSupportedConstraints.webidl',