Bug 1170958 - Allow MediaInputPort to lock to a specific input track. r=roc

Locking to specific tracks lets us dynamically remove and add single tracks to a ProcessedMediaStream. --HG-- extra : commitid : GPSNwBVyD4j extra : rebase_source : 0b1b79077f95bbefc8c71de551c5e3483a7d6ac0
2015-09-30 09:31:53 +08:00 · 2015-09-30 09:31:53 +08:00 · 53a6c38d0d
--- a/dom/media/AudioCaptureStream.cpp
+++ b/dom/media/AudioCaptureStream.cpp
@ -29,8 +29,8 @@ namespace mozilla
 // We are mixing to mono until PeerConnection can accept stereo
 static const uint32_t MONO = 1;

-AudioCaptureStream::AudioCaptureStream(DOMMediaStream* aWrapper)
-  : ProcessedMediaStream(aWrapper), mTrackCreated(false)
+AudioCaptureStream::AudioCaptureStream(DOMMediaStream* aWrapper, TrackID aTrackId)
+  : ProcessedMediaStream(aWrapper), mTrackId(aTrackId), mTrackCreated(false)
 {
  MOZ_ASSERT(NS_IsMainThread());
  MOZ_COUNT_CTOR(AudioCaptureStream);
@ -48,14 +48,14 @@ AudioCaptureStream::ProcessInput(GraphTime aFrom, GraphTime aTo,
                                 uint32_t aFlags)
 {
  uint32_t inputCount = mInputs.Length();
-  StreamBuffer::Track* track = EnsureTrack(AUDIO_TRACK);
+  StreamBuffer::Track* track = EnsureTrack(mTrackId);
  // Notify the DOM everything is in order.
  if (!mTrackCreated) {
    for (uint32_t i = 0; i < mListeners.Length(); i++) {
      MediaStreamListener* l = mListeners[i];
      AudioSegment tmp;
      l->NotifyQueuedTrackChanges(
-        Graph(), AUDIO_TRACK, 0, MediaStreamListener::TRACK_EVENT_CREATED, tmp);
+        Graph(), mTrackId, 0, MediaStreamListener::TRACK_EVENT_CREATED, tmp);
      l->NotifyFinishedTrackCreation(Graph());
    }
    mTrackCreated = true;
@ -127,6 +127,6 @@ AudioCaptureStream::MixerCallback(AudioDataValue* aMixedBuffer,
  }

  // Now we have mixed data, simply append it to out track.
-  EnsureTrack(AUDIO_TRACK)->Get<AudioSegment>()->AppendAndConsumeChunk(&chunk);
+  EnsureTrack(mTrackId)->Get<AudioSegment>()->AppendAndConsumeChunk(&chunk);
 }
 }
--- a/dom/media/AudioCaptureStream.h
+++ b/dom/media/AudioCaptureStream.h
@ -8,6 +8,7 @@

 #include "MediaStreamGraph.h"
 #include "AudioMixer.h"
+#include "StreamBuffer.h"
 #include <algorithm>

 namespace mozilla
@ -22,17 +23,17 @@ class AudioCaptureStream : public ProcessedMediaStream,
                           public MixerCallbackReceiver
 {
 public:
-  explicit AudioCaptureStream(DOMMediaStream* aWrapper);
+  explicit AudioCaptureStream(DOMMediaStream* aWrapper, TrackID aTrackId);
  virtual ~AudioCaptureStream();

  void ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) override;

 protected:
-  enum { AUDIO_TRACK = 1 };
  void MixerCallback(AudioDataValue* aMixedBuffer, AudioSampleFormat aFormat,
                     uint32_t aChannels, uint32_t aFrames,
                     uint32_t aSampleRate) override;
  AudioMixer mMixer;
+  TrackID mTrackId;
  bool mTrackCreated;
 };
 }
--- a/dom/media/DOMMediaStream.cpp
+++ b/dom/media/DOMMediaStream.cpp
@ -308,7 +308,10 @@ DOMMediaStream::InitAudioCaptureStream(nsIDOMWindow* aWindow,
 {
  mWindow = aWindow;

-  InitStreamCommon(aGraph->CreateAudioCaptureStream(this));
+  const TrackID AUDIO_TRACK = 1;
+
+  InitStreamCommon(aGraph->CreateAudioCaptureStream(this, AUDIO_TRACK));
+  CreateDOMTrack(AUDIO_TRACK, MediaSegment::AUDIO);
 }

 void
--- a/dom/media/MediaRecorder.cpp
+++ b/dom/media/MediaRecorder.cpp
@ -540,8 +540,7 @@ private:
    mTrackUnionStream->SetAutofinish(true);

    // Bind this Track Union Stream with Source Media.
-    mInputPort = mTrackUnionStream->AllocateInputPort(mRecorder->GetSourceMediaStream(),
-                                                      0);
+    mInputPort = mTrackUnionStream->AllocateInputPort(mRecorder->GetSourceMediaStream());

    DOMMediaStream* domStream = mRecorder->Stream();
    if (domStream) {
@ -800,7 +799,7 @@ MediaRecorder::MediaRecorder(AudioNode& aSrcAudioNode,
    AudioNodeStream* ns = aSrcAudioNode.GetStream();
    if (ns) {
      mInputPort = mPipeStream->AllocateInputPort(aSrcAudioNode.GetStream(),
-                                                  0, aSrcOutput);
+                                                  TRACK_ANY, 0, aSrcOutput);
    }
  }
  mAudioNode = &aSrcAudioNode;
--- a/dom/media/MediaStreamGraph.cpp
+++ b/dom/media/MediaStreamGraph.cpp
@ -2430,8 +2430,39 @@ MediaInputPort::SetGraphImpl(MediaStreamGraphImpl* aGraph)
  mGraph = aGraph;
 }

+void
+MediaInputPort::BlockTrackIdImpl(TrackID aTrackId)
+{
+  mBlockedTracks.AppendElement(aTrackId);
+}
+
+void
+MediaInputPort::BlockTrackId(TrackID aTrackId)
+{
+  class Message : public ControlMessage {
+  public:
+    explicit Message(MediaInputPort* aPort, TrackID aTrackId)
+      : ControlMessage(aPort->GetDestination()),
+        mPort(aPort), mTrackId(aTrackId) {}
+    virtual void Run()
+    {
+      mPort->BlockTrackIdImpl(mTrackId);
+    }
+    virtual void RunDuringShutdown()
+    {
+      Run();
+    }
+    nsRefPtr<MediaInputPort> mPort;
+    TrackID mTrackId;
+  };
+
+  MOZ_ASSERT(aTrackId != TRACK_NONE && aTrackId != TRACK_INVALID && aTrackId != TRACK_ANY,
+             "Only explicit TrackID is allowed");
+  GraphImpl()->AppendMessage(new Message(this, aTrackId));
+}
+
 already_AddRefed<MediaInputPort>
-ProcessedMediaStream::AllocateInputPort(MediaStream* aStream,
+ProcessedMediaStream::AllocateInputPort(MediaStream* aStream, TrackID aTrackID,
                                        uint16_t aInputNumber, uint16_t aOutputNumber)
 {
  // This method creates two references to the MediaInputPort: one for
@ -2454,7 +2485,10 @@ ProcessedMediaStream::AllocateInputPort(MediaStream* aStream,
    }
    nsRefPtr<MediaInputPort> mPort;
  };
-  nsRefPtr<MediaInputPort> port = new MediaInputPort(aStream, this,
+
+  MOZ_ASSERT(aTrackID != TRACK_NONE && aTrackID != TRACK_INVALID,
+             "Only TRACK_ANY and explicit ID are allowed");
+  nsRefPtr<MediaInputPort> port = new MediaInputPort(aStream, aTrackID, this,
                                                     aInputNumber, aOutputNumber);
  port->SetGraphImpl(GraphImpl());
  GraphImpl()->AppendMessage(new Message(port));
@ -2765,9 +2799,10 @@ MediaStreamGraph::CreateTrackUnionStream(DOMMediaStream* aWrapper)
 }

 ProcessedMediaStream*
-MediaStreamGraph::CreateAudioCaptureStream(DOMMediaStream* aWrapper)
+MediaStreamGraph::CreateAudioCaptureStream(DOMMediaStream* aWrapper,
+                                           TrackID aTrackId)
 {
-  AudioCaptureStream* stream = new AudioCaptureStream(aWrapper);
+  AudioCaptureStream* stream = new AudioCaptureStream(aWrapper, aTrackId);
  AddStream(stream);
  return stream;
 }
--- a/dom/media/MediaStreamGraph.h
+++ b/dom/media/MediaStreamGraph.h
@ -919,6 +919,10 @@ protected:
 * We make these refcounted so that stream-related messages with MediaInputPort*
 * pointers can be sent to the main thread safely.
 *
+ * A port can be locked to a specific track in the source stream, in which case
+ * only this track will be forwarded to the destination stream. TRACK_ANY
+ * can used to signal that all tracks shall be forwarded.
+ *
 * When a port's source or destination stream dies, the stream's DestroyImpl
 * calls MediaInputPort::Disconnect to disconnect the port from
 * the source and destination streams.
@ -934,9 +938,11 @@ class MediaInputPort final
 {
 private:
  // Do not call this constructor directly. Instead call aDest->AllocateInputPort.
-  MediaInputPort(MediaStream* aSource, ProcessedMediaStream* aDest,
+  MediaInputPort(MediaStream* aSource, TrackID& aSourceTrack,
+                 ProcessedMediaStream* aDest,
                 uint16_t aInputNumber, uint16_t aOutputNumber)
    : mSource(aSource)
+    , mSourceTrack(aSourceTrack)
    , mDest(aDest)
    , mInputNumber(aInputNumber)
    , mOutputNumber(aOutputNumber)
@ -969,8 +975,22 @@ public:

  // Any thread
  MediaStream* GetSource() { return mSource; }
+  TrackID GetSourceTrackId() { return mSourceTrack; }
  ProcessedMediaStream* GetDestination() { return mDest; }

+  // Block aTrackId in the port. Consumers will interpret this track as ended.
+  void BlockTrackId(TrackID aTrackId);
+private:
+  void BlockTrackIdImpl(TrackID aTrackId);
+
+public:
+  // Returns true if aTrackId has not been blocked and this port has not
+  // been locked to another track.
+  bool PassTrackThrough(TrackID aTrackId) {
+    return !mBlockedTracks.Contains(aTrackId) &&
+           (mSourceTrack == TRACK_ANY || mSourceTrack == aTrackId);
+  }
+
  uint16_t InputNumber() const { return mInputNumber; }
  uint16_t OutputNumber() const { return mOutputNumber; }

@ -1016,11 +1036,13 @@ private:
  friend class ProcessedMediaStream;
  // Never modified after Init()
  MediaStream* mSource;
+  TrackID mSourceTrack;
  ProcessedMediaStream* mDest;
  // The input and output numbers are optional, and are currently only used by
  // Web Audio.
  const uint16_t mInputNumber;
  const uint16_t mOutputNumber;
+  nsTArray<TrackID> mBlockedTracks;

  // Our media stream graph
  MediaStreamGraphImpl* mGraph;
@ -1042,8 +1064,13 @@ public:
  /**
   * Allocates a new input port attached to source aStream.
   * This stream can be removed by calling MediaInputPort::Remove().
+   * The input port is tied to aTrackID in the source stream.
+   * aTrackID can be set to TRACK_ANY to automatically forward all tracks from
+   * aStream. To end a track in the destination stream forwarded with TRACK_ANY,
+   * it can be blocked in the input port through MediaInputPort::BlockTrackId().
   */
  already_AddRefed<MediaInputPort> AllocateInputPort(MediaStream* aStream,
+                                                     TrackID aTrackID = TRACK_ANY,
                                                     uint16_t aInputNumber = 0,
                                                     uint16_t aOutputNumber = 0);
  /**
@ -1190,7 +1217,8 @@ public:
  /**
   * Create a stream that will mix all its audio input.
   */
-  ProcessedMediaStream* CreateAudioCaptureStream(DOMMediaStream* aWrapper);
+  ProcessedMediaStream* CreateAudioCaptureStream(DOMMediaStream* aWrapper,
+                                                 TrackID aTrackId);

  enum {
    ADD_STREAM_SUSPENDED = 0x01
--- a/dom/media/StreamBuffer.h
+++ b/dom/media/StreamBuffer.h
@ -19,6 +19,7 @@ namespace mozilla {
 typedef int32_t TrackID;
 const TrackID TRACK_NONE = 0;
 const TrackID TRACK_INVALID = -1;
+const TrackID TRACK_ANY = -2;

 inline TrackTicks RateConvertTicksRoundDown(TrackRate aOutRate,
                                            TrackRate aInRate,
--- a/dom/media/TrackUnionStream.cpp
+++ b/dom/media/TrackUnionStream.cpp
@ -96,18 +96,19 @@ TrackUnionStream::TrackUnionStream(DOMMediaStream* aWrapper) :
          if (map->mInputPort == mInputs[i] && map->mInputTrackID == tracks->GetID()) {
            bool trackFinished;
            StreamBuffer::Track* outputTrack = mBuffer.FindTrack(map->mOutputTrackID);
-            if (!outputTrack || outputTrack->IsEnded()) {
+            found = true;
+            if (!outputTrack || outputTrack->IsEnded() ||
+                !mInputs[i]->PassTrackThrough(tracks->GetID())) {
              trackFinished = true;
            } else {
              CopyTrackData(tracks.get(), j, aFrom, aTo, &trackFinished);
            }
            mappedTracksFinished[j] = trackFinished;
            mappedTracksWithMatchingInputTracks[j] = true;
-            found = true;
            break;
          }
        }
-        if (!found) {
+        if (!found && mInputs[i]->PassTrackThrough(tracks->GetID())) {
          bool trackFinished = false;
          trackAdded = true;
          uint32_t mapIndex = AddTrack(mInputs[i], tracks.get(), aFrom);
@ -191,7 +192,8 @@ TrackUnionStream::TrackUnionStream(DOMMediaStream* aWrapper) :
      MediaStreamListener* l = mListeners[j];
      l->NotifyQueuedTrackChanges(Graph(), id, outputStart,
                                  MediaStreamListener::TRACK_EVENT_CREATED,
-                                  *segment);
+                                  *segment,
+                                  aPort->GetSource(), aTrack->GetID());
    }
    segment->AppendNullData(outputStart);
    StreamBuffer::Track* track =
@ -223,7 +225,9 @@ TrackUnionStream::TrackUnionStream(DOMMediaStream* aWrapper) :
      segment = outputTrack->GetSegment()->CreateEmptyClone();
      l->NotifyQueuedTrackChanges(Graph(), outputTrack->GetID(), offset,
                                  MediaStreamListener::TRACK_EVENT_ENDED,
-                                  *segment);
+                                  *segment,
+                                  mTrackMap[aIndex].mInputPort->GetSource(),
+                                  mTrackMap[aIndex].mInputTrackID);
    }
    outputTrack->SetEnded();
  }
--- a/dom/media/webaudio/AudioNode.cpp
+++ b/dom/media/webaudio/AudioNode.cpp
@ -225,7 +225,8 @@ AudioNode::Connect(AudioNode& aDestination, uint32_t aOutput,
    MOZ_ASSERT(aInput <= UINT16_MAX, "Unexpected large input port number");
    MOZ_ASSERT(aOutput <= UINT16_MAX, "Unexpected large output port number");
    input->mStreamPort = destinationStream->
-      AllocateInputPort(mStream, static_cast<uint16_t>(aInput),
+      AllocateInputPort(mStream, AudioNodeStream::AUDIO_TRACK,
+                        static_cast<uint16_t>(aInput),
                        static_cast<uint16_t>(aOutput));
  }
  aDestination.NotifyInputsChanged();
@ -267,7 +268,8 @@ AudioNode::Connect(AudioParam& aDestination, uint32_t aOutput,
    // Setup our stream as an input to the AudioParam's stream
    MOZ_ASSERT(aOutput <= UINT16_MAX, "Unexpected large output port number");
    input->mStreamPort =
-      ps->AllocateInputPort(mStream, 0, static_cast<uint16_t>(aOutput));
+      ps->AllocateInputPort(mStream, AudioNodeStream::AUDIO_TRACK,
+                            0, static_cast<uint16_t>(aOutput));
  }
 }

--- a/dom/media/webaudio/AudioNodeExternalInputStream.cpp
+++ b/dom/media/webaudio/AudioNodeExternalInputStream.cpp
@ -142,6 +142,10 @@ AudioNodeExternalInputStream::ProcessInput(GraphTime aFrom, GraphTime aTo,
  for (StreamBuffer::TrackIter tracks(source->mBuffer, MediaSegment::AUDIO);
       !tracks.IsEnded(); tracks.Next()) {
    const StreamBuffer::Track& inputTrack = *tracks;
+    if (!mInputs[0]->PassTrackThrough(tracks->GetID())) {
+      continue;
+    }
+
    const AudioSegment& inputSegment =
        *static_cast<AudioSegment*>(inputTrack.GetSegment());
    if (inputSegment.IsNull()) {
--- a/dom/media/webaudio/AudioParam.cpp
+++ b/dom/media/webaudio/AudioParam.cpp
@ -114,7 +114,8 @@ AudioParam::Stream()
  // Setup the AudioParam's stream as an input to the owner AudioNode's stream
  AudioNodeStream* nodeStream = mNode->GetStream();
  if (nodeStream) {
-    mNodeStreamPort = nodeStream->AllocateInputPort(mStream);
+    mNodeStreamPort =
+      nodeStream->AllocateInputPort(mStream, AudioNodeStream::AUDIO_TRACK);
  }

  // Send the stream to the timeline on the MSG side.
--- a/dom/media/webaudio/MediaStreamAudioDestinationNode.cpp
+++ b/dom/media/webaudio/MediaStreamAudioDestinationNode.cpp
@ -41,7 +41,7 @@ MediaStreamAudioDestinationNode::MediaStreamAudioDestinationNode(AudioContext* a
  AudioNodeEngine* engine = new AudioNodeEngine(this);
  mStream = AudioNodeStream::Create(aContext, engine,
                                    AudioNodeStream::EXTERNAL_OUTPUT);
-  mPort = outputStream->AllocateInputPort(mStream);
+  mPort = outputStream->AllocateInputPort(mStream, AudioNodeStream::AUDIO_TRACK);

  nsIDocument* doc = aContext->GetParentObject()->GetExtantDoc();
  if (doc) {
--- a/dom/media/webaudio/MediaStreamAudioSourceNode.cpp
+++ b/dom/media/webaudio/MediaStreamAudioSourceNode.cpp
@ -8,6 +8,7 @@
 #include "mozilla/dom/MediaStreamAudioSourceNodeBinding.h"
 #include "AudioNodeEngine.h"
 #include "AudioNodeExternalInputStream.h"
+#include "AudioStreamTrack.h"
 #include "nsIDocument.h"
 #include "mozilla/CORSMode.h"

--- a/dom/media/webaudio/MediaStreamAudioSourceNode.h
+++ b/dom/media/webaudio/MediaStreamAudioSourceNode.h
@ -44,7 +44,8 @@ class MediaStreamAudioSourceNode : public AudioNode,
                                   public DOMMediaStream::PrincipalChangeObserver
 {
 public:
-  MediaStreamAudioSourceNode(AudioContext* aContext, DOMMediaStream* aMediaStream);
+  MediaStreamAudioSourceNode(AudioContext* aContext,
+                             DOMMediaStream* aMediaStream);

  NS_DECL_ISUPPORTS_INHERITED
  NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(MediaStreamAudioSourceNode, AudioNode)
--- a/dom/media/webspeech/synth/nsSpeechTask.cpp
+++ b/dom/media/webspeech/synth/nsSpeechTask.cpp
@ -21,6 +21,8 @@
 extern PRLogModuleInfo* GetSpeechSynthLog();
 #define LOG(type, msg) MOZ_LOG(GetSpeechSynthLog(), type, msg)

+#define AUDIO_TRACK 1
+
 namespace mozilla {
 namespace dom {

@ -193,7 +195,7 @@ nsSpeechTask::Setup(nsISpeechTaskCallback* aCallback,
  mChannels = aChannels;

  AudioSegment* segment = new AudioSegment();
-  mStream->AddAudioTrack(1, aRate, 0, segment);
+  mStream->AddAudioTrack(AUDIO_TRACK, aRate, 0, segment);
  mStream->AddAudioOutput(this);
  mStream->SetAudioOutputVolume(this, mVolume);