Bug 1221587: Base update of the MSG API for full-duplex r=padenot

--HG--
extra : commitid : Kf1JpZKH7LH

dom/media/GraphDriver.cpp

@@ -885,6 +885,23 @@ AudioCallbackDriver::DataCallback(AudioDataValue* aBuffer, long aFrames)
 
   mBuffer.BufferFilled();
 
+  // Callback any observers for the AEC speaker data.  Note that one
+  // (maybe) of these will be full-duplex, the others will get their input
+  // data off separate cubeb callbacks.  Take care with how stuff is
+  // removed/added to this list and TSAN issues, but input and output will
+  // use separate callback methods.
+  mGraphImpl->NotifySpeakerData(aOutputBuffer, static_cast<size_t>(aFrames),
+                                ChannelCount);
+
+  // Process mic data if any/needed -- after inserting far-end data for AEC!
+  if (aInputBuffer) {
+    if (mAudioInput) { // for this specific input-only or full-duplex stream
+      mAudioInput->NotifyInputData(mGraphImpl, aInputBuffer,
+                                   static_cast<size_t>(aFrames),
+                                   ChannelCount);
+    }
+  }
+
   bool switching = false;
   {
     MonitorAutoLock mon(mGraphImpl->GetMonitor());

dom/media/GraphDriver.h

@@ -196,6 +196,16 @@ public:
 
   virtual bool OnThread() = 0;
 
+  // XXX Thread-safety! Do these via commands to avoid TSAN issues
+  // and crashes!!!
+  virtual void SetInputListener(MediaStreamListener *aListener) {
+    mAudioInput = aListener;
+  }
+  // XXX do we need the param?  probably no
+  virtual void RemoveInputListener(MediaStreamListener *aListener) {
+    mAudioInput = nullptr;
+  }
+
 protected:
   GraphTime StateComputedTime() const;
 
@@ -226,6 +236,9 @@ protected:
   // This must be access with the monitor.
   WaitState mWaitState;
 
+  // Callback for mic data, if any
+  RefPtr<MediaStreamListener> mAudioInput;
+
   // This is used on the main thread (during initialization), and the graph
   // thread. No monitor needed because we know the graph thread does not run
   // during the initialization.
@@ -488,6 +501,8 @@ private:
    * This is synchronized by the Graph's monitor.
    * */
   bool mStarted;
+  /* Listener for mic input, if any. */
+  RefPtr<MediaStreamListener> mAudioInput;
 
   struct AutoInCallback
   {

dom/media/MediaStreamGraph.cpp

@@ -30,6 +30,7 @@
 #ifdef MOZ_WEBRTC
 #include "AudioOutputObserver.h"
 #endif
+#include "mtransport/runnable_utils.h"
 
 #include "webaudio/blink/HRTFDatabaseLoader.h"
 
@@ -922,6 +923,85 @@ MediaStreamGraphImpl::PlayVideo(MediaStream* aStream)
   }
 }
 
+void
+MediaStreamGraphImpl::OpenAudioInputImpl(char *aName, MediaStreamListener *aListener)
+{
+  if (CurrentDriver()->AsAudioCallbackDriver()) {
+    CurrentDriver()->SetInputListener(aListener);
+  } else {
+    // XXX Switch to callback driver
+  }
+  mAudioInputs.AppendElement(aListener); // always monitor speaker data
+}
+
+nsresult
+MediaStreamGraphImpl::OpenAudioInput(char *aName, MediaStreamListener *aListener)
+{
+  // XXX So, so, so annoying.  Can't AppendMessage except on Mainthread
+  if (!NS_IsMainThread()) {
+    NS_DispatchToMainThread(WrapRunnable(this,
+                                         &MediaStreamGraphImpl::OpenAudioInput,
+                                         aName, aListener)); // XXX Fix! string need to copied
+    return NS_OK;
+  }
+  class Message : public ControlMessage {
+  public:
+    Message(MediaStreamGraphImpl *aGraph, char *aName, MediaStreamListener *aListener) :
+      ControlMessage(nullptr), mGraph(aGraph), mName(aName), mListener(aListener) {}
+    virtual void Run()
+    {
+      mGraph->OpenAudioInputImpl(mName, mListener);
+    }
+    MediaStreamGraphImpl *mGraph;
+    char *mName; // XXX needs to copy
+    MediaStreamListener *mListener;
+  };
+  this->AppendMessage(new Message(this, aName, aListener));
+  return NS_OK;
+}
+
+void
+MediaStreamGraphImpl::CloseAudioInputImpl(MediaStreamListener *aListener)
+{
+  CurrentDriver()->RemoveInputListener(aListener);
+  mAudioInputs.RemoveElement(aListener);
+}
+
+void
+MediaStreamGraphImpl::CloseAudioInput(MediaStreamListener *aListener)
+{
+  // XXX So, so, so annoying.  Can't AppendMessage except on Mainthread
+  if (!NS_IsMainThread()) {
+    NS_DispatchToMainThread(WrapRunnable(this,
+                                         &MediaStreamGraphImpl::CloseAudioInput,
+                                         aListener));
+    return;
+  }
+  class Message : public ControlMessage {
+  public:
+    Message(MediaStreamGraphImpl *aGraph, MediaStreamListener *aListener) :
+      ControlMessage(nullptr), mGraph(aGraph), mListener(aListener) {}
+    virtual void Run()
+    {
+      mGraph->CloseAudioInputImpl(mListener);
+    }
+    MediaStreamGraphImpl *mGraph;
+    MediaStreamListener *mListener;
+  };
+  this->AppendMessage(new Message(this, aListener));
+}
+
+
+// All AudioInput listeners get the same speaker data (at least for now).
+void
+MediaStreamGraph::NotifySpeakerData(AudioDataValue* aBuffer, size_t aFrames,
+                                    uint32_t aChannels)
+{
+  for (auto& listener : mAudioInputs) {
+    listener->NotifySpeakerData(this, aBuffer, aFrames, aChannels);
+  }
+}
+
 bool
 MediaStreamGraphImpl::ShouldUpdateMainThread()
 {

dom/media/MediaStreamGraph.h

@@ -181,6 +181,23 @@ public:
    * are also notified of atomically to MediaStreamListeners.
    */
   virtual void NotifyFinishedTrackCreation(MediaStreamGraph* aGraph) {}
+
+  /* These are for cubeb audio input & output streams: */
+  /**
+   * Output data to speakers, for use as the "far-end" data for echo
+   * cancellation.  This is not guaranteed to be in any particular size
+   * chunks.
+   */
+  virtual void NotifySpeakerData(MediaStreamGraph* aGraph,
+                                 AudioDataValue* aBuffer, size_t aFrames,
+                                 uint32_t aChannels) {}
+  /**
+   * Input data from a microphone (or other audio source.  This is not
+   * guaranteed to be in any particular size chunks.
+   */
+  virtual void NotifyInputData(MediaStreamGraph* aGraph,
+                               AudioDataValue* aBuffer, size_t aFrames,
+                               uint32_t aChannels) {}
 };
 
 /**
@@ -1175,6 +1192,11 @@ public:
   // Idempotent
   static void DestroyNonRealtimeInstance(MediaStreamGraph* aGraph);
 
+  virtual nsresult OpenAudioInput(char *aName, MediaStreamListener *aListener) {
+    return NS_ERROR_FAILURE;
+  }
+  virtual void CloseAudioInput(MediaStreamListener *aListener) {}
+
   // Control API.
   /**
    * Create a stream that a media decoder (or some other source of
@@ -1254,6 +1276,13 @@ public:
   already_AddRefed<MediaInputPort> ConnectToCaptureStream(
     uint64_t aWindowId, MediaStream* aMediaStream);
 
+  /**
+   * Data going to the speakers from the GraphDriver's DataCallback
+   * to notify any listeners (for echo cancellation).
+   */
+  void NotifySpeakerData(AudioDataValue* aBuffer, size_t aFrames,
+                         uint32_t aChannels);
+
 protected:
   explicit MediaStreamGraph(TrackRate aSampleRate)
     : mSampleRate(aSampleRate)
@@ -1274,6 +1303,8 @@ protected:
    * at construction.
    */
   TrackRate mSampleRate;
+
+  nsTArray<RefPtr<MediaStreamListener>> mAudioInputs;
 };
 
 } // namespace mozilla

dom/media/MediaStreamGraphImpl.h

@@ -350,6 +350,11 @@ public:
    * at the current buffer end point. The StreamBuffer's tracks must be
    * explicitly set to finished by the caller.
    */
+  void OpenAudioInputImpl(char *aName, MediaStreamListener *aListener);
+  virtual nsresult OpenAudioInput(char *aName, MediaStreamListener *aListener) override;
+  void CloseAudioInputImpl(MediaStreamListener *aListener);
+  virtual void CloseAudioInput(MediaStreamListener *aListener) override;
+
   void FinishStream(MediaStream* aStream);
   /**
    * Compute how much stream data we would like to buffer for aStream.