Bug 1156472 - Part 3 - Implement AudioCaptureStream. r=roc

It is a ProcessMediaStream that simply mixes its inputs into a mono stream,
up/down mixing appropriately.

dom/media/AudioCaptureStream.cpp.rej

@@ -0,0 +1,133 @@
++/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
++/* This Source Code Form is subject to the terms of the Mozilla Public
++ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
++ * You can obtain one at http://mozilla.org/MPL/2.0/. */
++
++#include "MediaStreamGraphImpl.h"
++#include "mozilla/MathAlgorithms.h"
++#include "mozilla/unused.h"
++
++#include "AudioSegment.h"
++#include "mozilla/Logging.h"
++#include "mozilla/Attributes.h"
++#include "AudioCaptureStream.h"
++#include "ImageContainer.h"
++#include "AudioNodeEngine.h"
++#include "AudioNodeStream.h"
++#include "AudioNodeExternalInputStream.h"
++#include "webaudio/MediaStreamAudioDestinationNode.h"
++#include <algorithm>
++#include "DOMMediaStream.h"
++
++using namespace mozilla::layers;
++using namespace mozilla::dom;
++using namespace mozilla::gfx;
++
++namespace mozilla
++{
++
++// We are mixing to mono until PeerConnection can accept stereo
++static const uint32_t MONO = 1;
++
++AudioCaptureStream::AudioCaptureStream(DOMMediaStream* aWrapper)
++  : ProcessedMediaStream(aWrapper), mTrackCreated(false)
++{
++  MOZ_ASSERT(NS_IsMainThread());
++  MOZ_COUNT_CTOR(AudioCaptureStream);
++  mMixer.AddCallback(this);
++}
++
++AudioCaptureStream::~AudioCaptureStream()
++{
++  MOZ_COUNT_DTOR(AudioCaptureStream);
++  mMixer.RemoveCallback(this);
++}
++
++void
++AudioCaptureStream::ProcessInput(GraphTime aFrom, GraphTime aTo,
++                                 uint32_t aFlags)
++{
++  uint32_t inputCount = mInputs.Length();
++  StreamBuffer::Track* track = EnsureTrack(AUDIO_TRACK);
++  // Notify the DOM everything is in order.
++  if (!mTrackCreated) {
++    for (uint32_t i = 0; i < mListeners.Length(); i++) {
++      MediaStreamListener* l = mListeners[i];
++      AudioSegment tmp;
++      l->NotifyQueuedTrackChanges(
++        Graph(), AUDIO_TRACK, 0, MediaStreamListener::TRACK_EVENT_CREATED, tmp);
++      l->NotifyFinishedTrackCreation(Graph());
++    }
++    mTrackCreated = true;
++  }
++
++  // If the captured stream is connected back to a object on the page (be it an
++  // HTMLMediaElement with a stream as source, or an AudioContext), a cycle
++  // situation occur. This can work if it's an AudioContext with at least one
++  // DelayNode, but the MSG will mute the whole cycle otherwise.
++  bool blocked = mFinished || mBlocked.GetAt(aFrom);
++  if (blocked || InMutedCycle() || inputCount == 0) {
++    track->Get<AudioSegment>()->AppendNullData(aTo - aFrom);
++  } else {
++    // We mix down all the tracks of all inputs, to a stereo track. Everything
++    // is {up,down}-mixed to stereo.
++    mMixer.StartMixing();
++    AudioSegment output;
++    for (uint32_t i = 0; i < inputCount; i++) {
++      MediaStream* s = mInputs[i]->GetSource();
++      StreamBuffer::TrackIter tracks(s->GetStreamBuffer(), MediaSegment::AUDIO);
++      while (!tracks.IsEnded()) {
++        AudioSegment* inputSegment = tracks->Get<AudioSegment>();
++        StreamTime inputStart = s->GraphTimeToStreamTime(aFrom);
++        StreamTime inputEnd = s->GraphTimeToStreamTime(aTo);
++        AudioSegment toMix;
++        toMix.AppendSlice(*inputSegment, inputStart, inputEnd);
++        // Care for streams blocked in the [aTo, aFrom] range.
++        if (inputEnd - inputStart < aTo - aFrom) {
++          toMix.AppendNullData((aTo - aFrom) - (inputEnd - inputStart));
++        }
++        toMix.Mix(mMixer, MONO, Graph()->GraphRate());
++        tracks.Next();
++      }
++    }
++    // This calls MixerCallback below
++    mMixer.FinishMixing();
++  }
++
++  // Regardless of the status of the input tracks, we go foward.
++  mBuffer.AdvanceKnownTracksTime(GraphTimeToStreamTime((aTo)));
++}
++
++void
++AudioCaptureStream::MixerCallback(AudioDataValue* aMixedBuffer,
++                                  AudioSampleFormat aFormat, uint32_t aChannels,
++                                  uint32_t aFrames, uint32_t aSampleRate)
++{
++  nsAutoTArray<nsTArray<float>, MONO> output;
++  nsAutoTArray<const float*, MONO> bufferPtrs;
++  output.SetLength(MONO);
++  bufferPtrs.SetLength(MONO);
++
++  uint32_t written = 0;
++  // We need to copy here, because the mixer will reuse the storage, we should
++  // not hold onto it. Buffers are in planar format.
++  for (uint32_t channel = 0; channel < aChannels; channel++) {
++    float* out = output[channel].AppendElements(aFrames);
++    PodCopy(out, aMixedBuffer + written, aFrames);
++    bufferPtrs[channel] = out;
++    written += aFrames;
++  }
++  AudioChunk chunk;
++  chunk.mBuffer = new mozilla::SharedChannelArrayBuffer<float>(&output);
++  chunk.mDuration = aFrames;
++  chunk.mBufferFormat = AUDIO_FORMAT_FLOAT32;
++  chunk.mVolume = 1.0f;
++  chunk.mChannelData.SetLength(MONO);
++  for (uint32_t channel = 0; channel < aChannels; channel++) {
++    chunk.mChannelData[channel] = bufferPtrs[channel];
++  }
++
++  // Now we have mixed data, simply append it to out track.
++  EnsureTrack(AUDIO_TRACK)->Get<AudioSegment>()->AppendAndConsumeChunk(&chunk);
++}
++}

dom/media/AudioCaptureStream.h.rej

@@ -0,0 +1,43 @@
+--- AudioCaptureStream.h
++++ AudioCaptureStream.h
+@@ -0,0 +1,40 @@
++/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
++/* This Source Code Form is subject to the terms of the Mozilla Public
++ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
++ * You can obtain one at http://mozilla.org/MPL/2.0/. */
++
++#ifndef MOZILLA_AUDIOCAPTURESTREAM_H_
++#define MOZILLA_AUDIOCAPTURESTREAM_H_
++
++#include "MediaStreamGraph.h"
++#include "AudioMixer.h"
++#include <algorithm>
++
++namespace mozilla
++{
++
++class DOMMediaStream;
++
++/**
++ * See MediaStreamGraph::CreateAudioCaptureStream.
++ */
++class AudioCaptureStream : public ProcessedMediaStream,
++                           public MixerCallbackReceiver
++{
++public:
++  explicit AudioCaptureStream(DOMMediaStream* aWrapper);
++  virtual ~AudioCaptureStream();
++
++  void ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) override;
++
++protected:
++  enum { AUDIO_TRACK = 1 };
++  void MixerCallback(AudioDataValue* aMixedBuffer, AudioSampleFormat aFormat,
++                     uint32_t aChannels, uint32_t aFrames,
++                     uint32_t aSampleRate) override;
++  AudioMixer mMixer;
++  bool mTrackCreated;
++};
++}
++
++#endif /* MOZILLA_AUDIOCAPTURESTREAM_H_ */

dom/media/AudioMixer.h

@@ -26,7 +26,9 @@ struct MixerCallbackReceiver {
  * stream.
  *
  * AudioMixer::Mix is to be called repeatedly with buffers that have the same
- * length, sample rate, sample format and channel count.
+ * length, sample rate, sample format and channel count. This class works with
+ * interleaved and plannar buffers, but the buffer mixed must be of the same
+ * type during a mixing cycle.
  *
  * When all the tracks have been mixed, calling FinishMixing will call back with
  * a buffer containing the mixed audio data.
@@ -71,7 +73,7 @@ public:
     mSampleRate = mChannels = mFrames = 0;
   }
 
-  /* Add a buffer to the mix. aSamples is interleaved. */
+  /* Add a buffer to the mix. */
   void Mix(AudioDataValue* aSamples,
            uint32_t aChannels,
            uint32_t aFrames,

dom/media/AudioSegment.cpp

@@ -146,6 +146,103 @@ void AudioSegment::ResampleChunks(SpeexResamplerState* aResampler, uint32_t aInR
   }
 }
 
+// This helps to to safely get a pointer to the position we want to start
+// writing a planar audio buffer, depending on the channel and the offset in the
+// buffer.
+static AudioDataValue*
+PointerForOffsetInChannel(AudioDataValue* aData, size_t aLengthSamples,
+                          uint32_t aChannelCount, uint32_t aChannel,
+                          uint32_t aOffsetSamples)
+{
+  size_t samplesPerChannel = aLengthSamples / aChannelCount;
+  size_t beginningOfChannel = samplesPerChannel * aChannel;
+  MOZ_ASSERT(aChannel * samplesPerChannel + aOffsetSamples < aLengthSamples,
+             "Offset request out of bounds.");
+  return aData + beginningOfChannel + aOffsetSamples;
+}
+
+void
+AudioSegment::Mix(AudioMixer& aMixer, uint32_t aOutputChannels,
+                  uint32_t aSampleRate)
+{
+  nsAutoTArray<AudioDataValue, AUDIO_PROCESSING_FRAMES* GUESS_AUDIO_CHANNELS>
+  buf;
+  nsAutoTArray<const void*, GUESS_AUDIO_CHANNELS> channelData;
+  uint32_t offsetSamples = 0;
+  uint32_t duration = GetDuration();
+
+  if (duration <= 0) {
+    MOZ_ASSERT(duration == 0);
+    return;
+  }
+
+  uint32_t outBufferLength = duration * aOutputChannels;
+  buf.SetLength(outBufferLength);
+
+  for (ChunkIterator ci(*this); !ci.IsEnded(); ci.Next()) {
+    AudioChunk& c = *ci;
+    uint32_t frames = c.mDuration;
+
+    // If the chunk is silent, simply write the right number of silence in the
+    // buffers.
+    if (c.mBufferFormat == AUDIO_FORMAT_SILENCE) {
+      for (uint32_t channel = 0; channel < aOutputChannels; channel++) {
+        AudioDataValue* ptr =
+          PointerForOffsetInChannel(buf.Elements(), outBufferLength,
+                                    aOutputChannels, channel, offsetSamples);
+        PodZero(ptr, frames);
+      }
+    } else {
+      // Othewise, we need to upmix or downmix appropriately, depending on the
+      // desired input and output channels.
+      channelData.SetLength(c.mChannelData.Length());
+      for (uint32_t i = 0; i < channelData.Length(); ++i) {
+        channelData[i] = c.mChannelData[i];
+      }
+      if (channelData.Length() < aOutputChannels) {
+        // Up-mix.
+        AudioChannelsUpMix(&channelData, aOutputChannels, gZeroChannel);
+        for (uint32_t channel = 0; channel < aOutputChannels; channel++) {
+          AudioDataValue* ptr =
+            PointerForOffsetInChannel(buf.Elements(), outBufferLength,
+                                      aOutputChannels, channel, offsetSamples);
+          PodCopy(ptr, reinterpret_cast<const float*>(channelData[channel]),
+                  frames);
+        }
+        MOZ_ASSERT(channelData.Length() == aOutputChannels);
+      } else if (channelData.Length() > aOutputChannels) {
+        // Down mix.
+        nsAutoTArray<float*, GUESS_AUDIO_CHANNELS> outChannelPtrs;
+        outChannelPtrs.SetLength(aOutputChannels);
+        uint32_t offsetSamples = 0;
+        for (uint32_t channel = 0; channel < aOutputChannels; channel++) {
+          outChannelPtrs[channel] =
+            PointerForOffsetInChannel(buf.Elements(), outBufferLength,
+                                      aOutputChannels, channel, offsetSamples);
+        }
+        AudioChannelsDownMix(channelData, outChannelPtrs.Elements(),
+                             aOutputChannels, frames);
+      } else {
+        // The channel count is already what we want, just copy it over.
+        for (uint32_t channel = 0; channel < aOutputChannels; channel++) {
+          AudioDataValue* ptr =
+            PointerForOffsetInChannel(buf.Elements(), outBufferLength,
+                                      aOutputChannels, channel, offsetSamples);
+          PodCopy(ptr, reinterpret_cast<const float*>(channelData[channel]),
+                  frames);
+        }
+      }
+    }
+    offsetSamples += frames;
+  }
+
+  if (offsetSamples) {
+    MOZ_ASSERT(offsetSamples == outBufferLength / aOutputChannels,
+               "We forgot to write some samples?");
+    aMixer.Mix(buf.Elements(), aOutputChannels, offsetSamples, aSampleRate);
+  }
+}
+
 void
 AudioSegment::WriteTo(uint64_t aID, AudioMixer& aMixer, uint32_t aOutputChannels, uint32_t aSampleRate)
 {

dom/media/AudioSegment.h

@@ -299,7 +299,14 @@ public:
     return chunk;
   }
   void ApplyVolume(float aVolume);
-  void WriteTo(uint64_t aID, AudioMixer& aMixer, uint32_t aChannelCount, uint32_t aSampleRate);
+  // Mix the segment into a mixer, interleaved. This is useful to output a
+  // segment to a system audio callback. It up or down mixes to aChannelCount
+  // channels.
+  void WriteTo(uint64_t aID, AudioMixer& aMixer, uint32_t aChannelCount,
+               uint32_t aSampleRate);
+  // Mix the segment into a mixer, keeping it planar, up or down mixing to
+  // aChannelCount channels.
+  void Mix(AudioMixer& aMixer, uint32_t aChannelCount, uint32_t aSampleRate);
 
   int ChannelCount() {
     NS_WARN_IF_FALSE(!mChunks.IsEmpty(),

dom/media/DOMMediaStream.cpp

@@ -301,6 +301,18 @@ DOMMediaStream::InitTrackUnionStream(nsIDOMWindow* aWindow,
   InitStreamCommon(aGraph->CreateTrackUnionStream(this));
 }
 
+void
+DOMMediaStream::InitAudioCaptureStream(nsIDOMWindow* aWindow,
+                                       MediaStreamGraph* aGraph)
+{
+  mWindow = aWindow;
+
+  if (!aGraph) {
+    aGraph = MediaStreamGraph::GetInstance();
+  }
+  InitStreamCommon(aGraph->CreateAudioCaptureStream(this));
+}
+
 void
 DOMMediaStream::InitStreamCommon(MediaStream* aStream)
 {
@@ -329,6 +341,15 @@ DOMMediaStream::CreateTrackUnionStream(nsIDOMWindow* aWindow,
   return stream.forget();
 }
 
+already_AddRefed<DOMMediaStream>
+DOMMediaStream::CreateAudioCaptureStream(nsIDOMWindow* aWindow,
+                                         MediaStreamGraph* aGraph)
+{
+  nsRefPtr<DOMMediaStream> stream = new DOMMediaStream();
+  stream->InitAudioCaptureStream(aWindow, aGraph);
+  return stream.forget();
+}
+
 void
 DOMMediaStream::SetTrackEnabled(TrackID aTrackID, bool aEnabled)
 {
@@ -653,6 +674,15 @@ DOMLocalMediaStream::CreateTrackUnionStream(nsIDOMWindow* aWindow,
   return stream.forget();
 }
 
+already_AddRefed<DOMLocalMediaStream>
+DOMLocalMediaStream::CreateAudioCaptureStream(nsIDOMWindow* aWindow,
+                                              MediaStreamGraph* aGraph)
+{
+  nsRefPtr<DOMLocalMediaStream> stream = new DOMLocalMediaStream();
+  stream->InitAudioCaptureStream(aWindow, aGraph);
+  return stream.forget();
+}
+
 DOMAudioNodeMediaStream::DOMAudioNodeMediaStream(AudioNode* aNode)
 : mStreamNode(aNode)
 {

dom/media/DOMMediaStream.h

@@ -198,6 +198,13 @@ public:
   static already_AddRefed<DOMMediaStream> CreateTrackUnionStream(nsIDOMWindow* aWindow,
                                                                  MediaStreamGraph* aGraph = nullptr);
 
+  /**
+   * Create an nsDOMMediaStream whose underlying stream is an
+   * AudioCaptureStream
+   */
+  static already_AddRefed<DOMMediaStream> CreateAudioCaptureStream(
+    nsIDOMWindow* aWindow, MediaStreamGraph* aGraph = nullptr);
+
   void SetLogicalStreamStartTime(StreamTime aTime)
   {
     mLogicalStreamStartTime = aTime;
@@ -261,6 +268,8 @@ protected:
                         MediaStreamGraph* aGraph = nullptr);
   void InitTrackUnionStream(nsIDOMWindow* aWindow,
                             MediaStreamGraph* aGraph = nullptr);
+  void InitAudioCaptureStream(nsIDOMWindow* aWindow,
+                              MediaStreamGraph* aGraph = nullptr);
   void InitStreamCommon(MediaStream* aStream);
   already_AddRefed<AudioTrack> CreateAudioTrack(AudioStreamTrack* aStreamTrack);
   already_AddRefed<VideoTrack> CreateVideoTrack(VideoStreamTrack* aStreamTrack);
@@ -351,6 +360,12 @@ public:
   CreateTrackUnionStream(nsIDOMWindow* aWindow,
                          MediaStreamGraph* aGraph = nullptr);
 
+  /**
+   * Create an nsDOMLocalMediaStream whose underlying stream is an
+   * AudioCaptureStream. */
+  static already_AddRefed<DOMLocalMediaStream> CreateAudioCaptureStream(
+    nsIDOMWindow* aWindow, MediaStreamGraph* aGraph = nullptr);
+
 protected:
   virtual ~DOMLocalMediaStream();
 };

dom/media/moz.build

@@ -196,6 +196,7 @@ EXPORTS.mozilla.dom += [
 
 UNIFIED_SOURCES += [
     'AbstractThread.cpp',
+    'AudioCaptureStream.cpp',
     'AudioChannelFormat.cpp',
     'AudioCompactor.cpp',
     'AudioSegment.cpp',