diff --git a/dom/media/GraphDriver.cpp b/dom/media/GraphDriver.cpp index a57809e52e87..28fe738a955a 100644 --- a/dom/media/GraphDriver.cpp +++ b/dom/media/GraphDriver.cpp @@ -885,6 +885,23 @@ AudioCallbackDriver::DataCallback(AudioDataValue* aBuffer, long aFrames) mBuffer.BufferFilled(); + // Callback any observers for the AEC speaker data. Note that one + // (maybe) of these will be full-duplex, the others will get their input + // data off separate cubeb callbacks. Take care with how stuff is + // removed/added to this list and TSAN issues, but input and output will + // use separate callback methods. + mGraphImpl->NotifySpeakerData(aOutputBuffer, static_cast(aFrames), + ChannelCount); + + // Process mic data if any/needed -- after inserting far-end data for AEC! + if (aInputBuffer) { + if (mAudioInput) { // for this specific input-only or full-duplex stream + mAudioInput->NotifyInputData(mGraphImpl, aInputBuffer, + static_cast(aFrames), + ChannelCount); + } + } + bool switching = false; { MonitorAutoLock mon(mGraphImpl->GetMonitor()); diff --git a/dom/media/GraphDriver.h b/dom/media/GraphDriver.h index 0f626da4c033..7359f79409a7 100644 --- a/dom/media/GraphDriver.h +++ b/dom/media/GraphDriver.h @@ -196,6 +196,16 @@ public: virtual bool OnThread() = 0; + // XXX Thread-safety! Do these via commands to avoid TSAN issues + // and crashes!!! + virtual void SetInputListener(MediaStreamListener *aListener) { + mAudioInput = aListener; + } + // XXX do we need the param? probably no + virtual void RemoveInputListener(MediaStreamListener *aListener) { + mAudioInput = nullptr; + } + protected: GraphTime StateComputedTime() const; @@ -226,6 +236,9 @@ protected: // This must be access with the monitor. WaitState mWaitState; + // Callback for mic data, if any + RefPtr mAudioInput; + // This is used on the main thread (during initialization), and the graph // thread. No monitor needed because we know the graph thread does not run // during the initialization. @@ -488,6 +501,8 @@ private: * This is synchronized by the Graph's monitor. * */ bool mStarted; + /* Listener for mic input, if any. */ + RefPtr mAudioInput; struct AutoInCallback { diff --git a/dom/media/MediaStreamGraph.cpp b/dom/media/MediaStreamGraph.cpp index 417effba1e23..9f0ea42f9923 100644 --- a/dom/media/MediaStreamGraph.cpp +++ b/dom/media/MediaStreamGraph.cpp @@ -30,6 +30,7 @@ #ifdef MOZ_WEBRTC #include "AudioOutputObserver.h" #endif +#include "mtransport/runnable_utils.h" #include "webaudio/blink/HRTFDatabaseLoader.h" @@ -922,6 +923,85 @@ MediaStreamGraphImpl::PlayVideo(MediaStream* aStream) } } +void +MediaStreamGraphImpl::OpenAudioInputImpl(char *aName, MediaStreamListener *aListener) +{ + if (CurrentDriver()->AsAudioCallbackDriver()) { + CurrentDriver()->SetInputListener(aListener); + } else { + // XXX Switch to callback driver + } + mAudioInputs.AppendElement(aListener); // always monitor speaker data +} + +nsresult +MediaStreamGraphImpl::OpenAudioInput(char *aName, MediaStreamListener *aListener) +{ + // XXX So, so, so annoying. Can't AppendMessage except on Mainthread + if (!NS_IsMainThread()) { + NS_DispatchToMainThread(WrapRunnable(this, + &MediaStreamGraphImpl::OpenAudioInput, + aName, aListener)); // XXX Fix! string need to copied + return NS_OK; + } + class Message : public ControlMessage { + public: + Message(MediaStreamGraphImpl *aGraph, char *aName, MediaStreamListener *aListener) : + ControlMessage(nullptr), mGraph(aGraph), mName(aName), mListener(aListener) {} + virtual void Run() + { + mGraph->OpenAudioInputImpl(mName, mListener); + } + MediaStreamGraphImpl *mGraph; + char *mName; // XXX needs to copy + MediaStreamListener *mListener; + }; + this->AppendMessage(new Message(this, aName, aListener)); + return NS_OK; +} + +void +MediaStreamGraphImpl::CloseAudioInputImpl(MediaStreamListener *aListener) +{ + CurrentDriver()->RemoveInputListener(aListener); + mAudioInputs.RemoveElement(aListener); +} + +void +MediaStreamGraphImpl::CloseAudioInput(MediaStreamListener *aListener) +{ + // XXX So, so, so annoying. Can't AppendMessage except on Mainthread + if (!NS_IsMainThread()) { + NS_DispatchToMainThread(WrapRunnable(this, + &MediaStreamGraphImpl::CloseAudioInput, + aListener)); + return; + } + class Message : public ControlMessage { + public: + Message(MediaStreamGraphImpl *aGraph, MediaStreamListener *aListener) : + ControlMessage(nullptr), mGraph(aGraph), mListener(aListener) {} + virtual void Run() + { + mGraph->CloseAudioInputImpl(mListener); + } + MediaStreamGraphImpl *mGraph; + MediaStreamListener *mListener; + }; + this->AppendMessage(new Message(this, aListener)); +} + + +// All AudioInput listeners get the same speaker data (at least for now). +void +MediaStreamGraph::NotifySpeakerData(AudioDataValue* aBuffer, size_t aFrames, + uint32_t aChannels) +{ + for (auto& listener : mAudioInputs) { + listener->NotifySpeakerData(this, aBuffer, aFrames, aChannels); + } +} + bool MediaStreamGraphImpl::ShouldUpdateMainThread() { diff --git a/dom/media/MediaStreamGraph.h b/dom/media/MediaStreamGraph.h index bc8576fd0565..c1bc22c8336d 100644 --- a/dom/media/MediaStreamGraph.h +++ b/dom/media/MediaStreamGraph.h @@ -181,6 +181,23 @@ public: * are also notified of atomically to MediaStreamListeners. */ virtual void NotifyFinishedTrackCreation(MediaStreamGraph* aGraph) {} + + /* These are for cubeb audio input & output streams: */ + /** + * Output data to speakers, for use as the "far-end" data for echo + * cancellation. This is not guaranteed to be in any particular size + * chunks. + */ + virtual void NotifySpeakerData(MediaStreamGraph* aGraph, + AudioDataValue* aBuffer, size_t aFrames, + uint32_t aChannels) {} + /** + * Input data from a microphone (or other audio source. This is not + * guaranteed to be in any particular size chunks. + */ + virtual void NotifyInputData(MediaStreamGraph* aGraph, + AudioDataValue* aBuffer, size_t aFrames, + uint32_t aChannels) {} }; /** @@ -1175,6 +1192,11 @@ public: // Idempotent static void DestroyNonRealtimeInstance(MediaStreamGraph* aGraph); + virtual nsresult OpenAudioInput(char *aName, MediaStreamListener *aListener) { + return NS_ERROR_FAILURE; + } + virtual void CloseAudioInput(MediaStreamListener *aListener) {} + // Control API. /** * Create a stream that a media decoder (or some other source of @@ -1254,6 +1276,13 @@ public: already_AddRefed ConnectToCaptureStream( uint64_t aWindowId, MediaStream* aMediaStream); + /** + * Data going to the speakers from the GraphDriver's DataCallback + * to notify any listeners (for echo cancellation). + */ + void NotifySpeakerData(AudioDataValue* aBuffer, size_t aFrames, + uint32_t aChannels); + protected: explicit MediaStreamGraph(TrackRate aSampleRate) : mSampleRate(aSampleRate) @@ -1274,6 +1303,8 @@ protected: * at construction. */ TrackRate mSampleRate; + + nsTArray> mAudioInputs; }; } // namespace mozilla diff --git a/dom/media/MediaStreamGraphImpl.h b/dom/media/MediaStreamGraphImpl.h index c5e850ca6b3c..767e75a3f355 100644 --- a/dom/media/MediaStreamGraphImpl.h +++ b/dom/media/MediaStreamGraphImpl.h @@ -350,6 +350,11 @@ public: * at the current buffer end point. The StreamBuffer's tracks must be * explicitly set to finished by the caller. */ + void OpenAudioInputImpl(char *aName, MediaStreamListener *aListener); + virtual nsresult OpenAudioInput(char *aName, MediaStreamListener *aListener) override; + void CloseAudioInputImpl(MediaStreamListener *aListener); + virtual void CloseAudioInput(MediaStreamListener *aListener) override; + void FinishStream(MediaStream* aStream); /** * Compute how much stream data we would like to buffer for aStream.