Bug 849918 - Initial support for PannerNode's 3D positional audio (equalpower panning model). r=ehsan,roc

content/media/AudioNodeEngine.cpp

@@ -78,4 +78,48 @@ AudioBlockCopyChannelWithScale(const float aInput[WEBAUDIO_BLOCK_SIZE],
   }
 }
 
+void
+AudioBlockInPlaceScale(float aBlock[WEBAUDIO_BLOCK_SIZE],
+                       uint32_t aChannelCount,
+                       float aScale)
+{
+  if (aScale == 1.0f) {
+    return;
+  }
+  for (uint32_t i = 0; i < WEBAUDIO_BLOCK_SIZE * aChannelCount; ++i) {
+    *aBlock++ *= aScale;
+  }
+}
+
+void
+AudioBlockPanMonoToStereo(const float aInput[WEBAUDIO_BLOCK_SIZE],
+                          float aGainL, float aGainR,
+                          float aOutputL[WEBAUDIO_BLOCK_SIZE],
+                          float aOutputR[WEBAUDIO_BLOCK_SIZE])
+{
+  AudioBlockCopyChannelWithScale(aInput, aGainL, aOutputL);
+  AudioBlockCopyChannelWithScale(aInput, aGainR, aOutputR);
+}
+
+void
+AudioBlockPanStereoToStereo(const float aInputL[WEBAUDIO_BLOCK_SIZE],
+                            const float aInputR[WEBAUDIO_BLOCK_SIZE],
+                            float aGainL, float aGainR, bool aIsOnTheLeft,
+                            float aOutputL[WEBAUDIO_BLOCK_SIZE],
+                            float aOutputR[WEBAUDIO_BLOCK_SIZE])
+{
+  uint32_t i;
+
+  if (aIsOnTheLeft) {
+    for (i = 0; i < WEBAUDIO_BLOCK_SIZE; ++i) {
+      *aOutputL++ = *aInputL++ + *aInputR * aGainL;
+      *aOutputR++ = *aInputR++ * aGainR;
+    }
+  } else {
+    for (i = 0; i < WEBAUDIO_BLOCK_SIZE; ++i) {
+      *aOutputL++ = *aInputL * aGainL;
+      *aOutputR++ = *aInputR++ + *aInputL++ * aGainR;
+    }
+  }
+}
 }

content/media/AudioNodeEngine.h

@@ -111,6 +111,35 @@ void AudioBlockCopyChannelWithScale(const float aInput[WEBAUDIO_BLOCK_SIZE],
                                     const float aScale[WEBAUDIO_BLOCK_SIZE],
                                     float aOutput[WEBAUDIO_BLOCK_SIZE]);
 
+/**
+ * In place gain. aScale == 1.0f should be optimized.
+ */
+void AudioBlockInPlaceScale(float aBlock[WEBAUDIO_BLOCK_SIZE],
+                            uint32_t aChannelCount,
+                            float aScale);
+
+/**
+ * Upmix a mono input to a stereo output, scaling the two output channels by two
+ * different gain value.
+ * This algorithm is specified in the WebAudio spec.
+ */
+void
+AudioBlockPanMonoToStereo(const float aInput[WEBAUDIO_BLOCK_SIZE],
+                          float aGainL, float aGainR,
+                          float aOutputL[WEBAUDIO_BLOCK_SIZE],
+                          float aOutputR[WEBAUDIO_BLOCK_SIZE]);
+/**
+ * Pan a stereo source according to right and left gain, and the position
+ * (whether the listener is on the left of the source or not).
+ * This algorithm is specified in the WebAudio spec.
+ */
+void
+AudioBlockPanStereoToStereo(const float aInputL[WEBAUDIO_BLOCK_SIZE],
+                            const float aInputR[WEBAUDIO_BLOCK_SIZE],
+                            float aGainL, float aGainR, bool aIsOnTheLeft,
+                            float aOutputL[WEBAUDIO_BLOCK_SIZE],
+                            float aOutputR[WEBAUDIO_BLOCK_SIZE]);
+
 /**
  * All methods of this class and its subclasses are called on the
  * MediaStreamGraph thread.

content/media/webaudio/AudioListener.h

@@ -121,6 +121,7 @@ private:
   void SendThreeDPointParameterToStream(uint32_t aIndex, const ThreeDPoint& aValue);
 
 private:
+  friend class PannerNode;
   nsRefPtr<AudioContext> mContext;
   ThreeDPoint mPosition;
   ThreeDPoint mOrientation;

content/media/webaudio/Makefile.in

@@ -31,6 +31,7 @@ CPPSRCS := \
   GainNode.cpp \
   MediaBufferDecoder.cpp \
   PannerNode.cpp \
+  ThreeDPoint.cpp \
   $(NULL)
 
 EXPORTS_NAMESPACES := mozilla/dom

content/media/webaudio/PannerNode.cpp

@@ -12,13 +12,17 @@
 namespace mozilla {
 namespace dom {
 
+using namespace std;
+
 class PannerNodeEngine : public AudioNodeEngine
 {
 public:
   PannerNodeEngine()
     // Please keep these default values consistent with PannerNode::PannerNode below.
     : mPanningModel(PanningModelTypeValues::HRTF)
+    , mPanningModelFunction(&PannerNodeEngine::HRTFPanningFunction)
     , mDistanceModel(DistanceModelTypeValues::Inverse)
+    , mDistanceModelFunction(&PannerNodeEngine::InverseGainFunction)
     , mPosition()
     , mOrientation(1., 0., 0.)
     , mVelocity()
@@ -40,9 +44,31 @@ public:
     switch (aIndex) {
     case PannerNode::PANNING_MODEL:
       mPanningModel = PanningModelType(aParam);
+      switch (mPanningModel) {
+        case PanningModelTypeValues::Equalpower:
+          mPanningModelFunction = &PannerNodeEngine::EqualPowerPanningFunction;
+          break;
+        case PanningModelTypeValues::HRTF:
+          mPanningModelFunction = &PannerNodeEngine::HRTFPanningFunction;
+          break;
+        case PanningModelTypeValues::Soundfield:
+          mPanningModelFunction = &PannerNodeEngine::SoundfieldPanningFunction;
+          break;
+      }
       break;
     case PannerNode::DISTANCE_MODEL:
       mDistanceModel = DistanceModelType(aParam);
+      switch (mDistanceModel) {
+        case DistanceModelTypeValues::Inverse:
+          mDistanceModelFunction = &PannerNodeEngine::InverseGainFunction;
+          break;
+        case DistanceModelTypeValues::Linear:
+          mDistanceModelFunction = &PannerNodeEngine::LinearGainFunction;
+          break;
+        case DistanceModelTypeValues::Exponential:
+          mDistanceModelFunction = &PannerNodeEngine::ExponentialGainFunction;
+          break;
+      }
       break;
     default:
       NS_ERROR("Bad PannerNodeEngine Int32Parameter");
@@ -83,12 +109,35 @@ public:
                                  AudioChunk* aOutput,
                                  bool *aFinished) MOZ_OVERRIDE
   {
-    // TODO: actually do 3D positioning computations here
+    if (aInput.IsNull()) {
       *aOutput = aInput;
+      return;
+    }
+    (this->*mPanningModelFunction)(aInput, aOutput);
   }
 
+  void ComputeAzimuthAndElevation(float& aAzimuth, float& aElevation);
+  void DistanceGain(AudioChunk* aChunk, float aGain);
+
+  void GainMonoToStereo(const AudioChunk& aInput, AudioChunk* aOutput,
+                        float aGainL, float aGainR);
+  void GainStereoToStereo(const AudioChunk& aInput, AudioChunk* aOutput,
+                          float aGainL, float aGainR, double aAzimuth);
+
+  void EqualPowerPanningFunction(const AudioChunk& aInput, AudioChunk* aOutput);
+  void HRTFPanningFunction(const AudioChunk& aInput, AudioChunk* aOutput);
+  void SoundfieldPanningFunction(const AudioChunk& aInput, AudioChunk* aOutput);
+
+  float LinearGainFunction(float aDistance);
+  float InverseGainFunction(float aDistance);
+  float ExponentialGainFunction(float aDistance);
+
   PanningModelType mPanningModel;
+  typedef void (PannerNodeEngine::*PanningModelFunction)(const AudioChunk& aInput, AudioChunk* aOutput);
+  PanningModelFunction mPanningModelFunction;
   DistanceModelType mDistanceModel;
+  typedef float (PannerNodeEngine::*DistanceModelFunction)(float aDistance);
+  DistanceModelFunction mDistanceModelFunction;
   ThreeDPoint mPosition;
   ThreeDPoint mOrientation;
   ThreeDPoint mVelocity;
@@ -138,6 +187,190 @@ PannerNode::WrapObject(JSContext* aCx, JSObject* aScope)
   return PannerNodeBinding::Wrap(aCx, aScope, this);
 }
 
+// Those three functions are described in the spec.
+float
+PannerNodeEngine::LinearGainFunction(float aDistance)
+{
+  return 1 - mRolloffFactor * (aDistance - mRefDistance) / (mMaxDistance - mRefDistance);
+}
+
+float
+PannerNodeEngine::InverseGainFunction(float aDistance)
+{
+  return mRefDistance / (mRefDistance + mRolloffFactor * (aDistance - mRefDistance));
+}
+
+float
+PannerNodeEngine::ExponentialGainFunction(float aDistance)
+{
+  return pow(aDistance / mRefDistance, -mRolloffFactor);
+}
+
+void
+PannerNodeEngine::SoundfieldPanningFunction(const AudioChunk& aInput,
+                                            AudioChunk* aOutput)
+{
+  // not implemented: noop
+  *aOutput = aInput;
+}
+
+void
+PannerNodeEngine::HRTFPanningFunction(const AudioChunk& aInput,
+                                      AudioChunk* aOutput)
+{
+  // not implemented: noop
+  *aOutput = aInput;
+}
+
+void
+PannerNodeEngine::EqualPowerPanningFunction(const AudioChunk& aInput,
+                                            AudioChunk* aOutput)
+{
+  float azimuth, elevation, gainL, gainR, normalizedAzimuth, distance, distanceGain;
+  int inputChannels = aInput.mChannelData.Length();
+  ThreeDPoint distanceVec;
+
+  // If both the listener are in the same spot, and no cone gain is specified,
+  // this node is noop.
+  if (mListenerPosition == mPosition &&
+      mConeInnerAngle == 360 &&
+      mConeOuterAngle == 360) {
+    *aOutput = aInput;
+  }
+
+  // The output of this node is always stereo, no matter what the inputs are.
+  AllocateAudioBlock(2, aOutput);
+
+  ComputeAzimuthAndElevation(azimuth, elevation);
+
+  // The following algorithm is described in the spec.
+  // Clamp azimuth in the [-90, 90] range.
+  azimuth = min(180.f, max(-180.f, azimuth));
+
+  // Wrap around
+  if (azimuth < -90.f) {
+    azimuth = -180.f - azimuth;
+  } else if (azimuth > 90) {
+    azimuth = 180.f - azimuth;
+  }
+
+  // Normalize the value in the [0, 1] range.
+  if (inputChannels == 1) {
+    normalizedAzimuth = (azimuth + 90.f) / 180.f;
+  } else {
+    if (azimuth <= 0) {
+      normalizedAzimuth = (azimuth + 90.f) / 90.f;
+    } else {
+      normalizedAzimuth = azimuth / 90.f;
+    }
+  }
+
+  // Compute how much the distance contributes to the gain reduction.
+  distanceVec = mPosition - mListenerPosition;
+  distance = sqrt(distanceVec.DotProduct(distanceVec));
+  distanceGain = (this->*mDistanceModelFunction)(distance);
+
+  // Actually compute the left and right gain.
+  gainL = cos(0.5 * M_PI * normalizedAzimuth);
+  gainR = sin(0.5 * M_PI * normalizedAzimuth);
+
+  // Compute the output.
+  if (inputChannels == 1) {
+    GainMonoToStereo(aInput, aOutput, gainL, gainR);
+  } else {
+    GainStereoToStereo(aInput, aOutput, gainL, gainR, azimuth);
+  }
+
+  DistanceGain(aOutput, distanceGain);
+}
+
+void
+PannerNodeEngine::GainMonoToStereo(const AudioChunk& aInput, AudioChunk* aOutput,
+                                   float aGainL, float aGainR)
+{
+  float* outputL = static_cast<float*>(const_cast<void*>(aOutput->mChannelData[0]));
+  float* outputR = static_cast<float*>(const_cast<void*>(aOutput->mChannelData[1]));
+  const float* input = static_cast<float*>(const_cast<void*>(aInput.mChannelData[0]));
+
+  AudioBlockPanMonoToStereo(input, aGainL, aGainR, outputL, outputR);
+}
+
+void
+PannerNodeEngine::GainStereoToStereo(const AudioChunk& aInput, AudioChunk* aOutput,
+                                     float aGainL, float aGainR, double aAzimuth)
+{
+  float* outputL = static_cast<float*>(const_cast<void*>(aOutput->mChannelData[0]));
+  float* outputR = static_cast<float*>(const_cast<void*>(aOutput->mChannelData[1]));
+  const float* inputL = static_cast<float*>(const_cast<void*>(aInput.mChannelData[0]));
+  const float* inputR = static_cast<float*>(const_cast<void*>(aInput.mChannelData[1]));
+
+  AudioBlockPanStereoToStereo(inputL, inputR, aGainL, aGainR, aAzimuth <= 0, outputL, outputR);
+}
+
+void
+PannerNodeEngine::DistanceGain(AudioChunk* aChunk, float aGain)
+{
+  float* samples = static_cast<float*>(const_cast<void*>(*aChunk->mChannelData.Elements()));
+  uint32_t channelCount = aChunk->mChannelData.Length();
+
+  AudioBlockInPlaceScale(samples, channelCount, aGain);
+}
+
+// This algorithm is specicied in the webaudio spec.
+void
+PannerNodeEngine::ComputeAzimuthAndElevation(float& aAzimuth, float& aElevation)
+{
+  ThreeDPoint sourceListener = mPosition - mListenerPosition;
+
+  if (sourceListener.IsZero()) {
+    aAzimuth = 0.0;
+    aElevation = 0.0;
+    return;
+  }
+
+  sourceListener.Normalize();
+
+  // Project the source-listener vector on the x-z plane.
+  ThreeDPoint& listenerFront = mListenerOrientation;
+  ThreeDPoint listenerRightNorm = listenerFront.CrossProduct(mListenerUpVector);
+  listenerRightNorm.Normalize();
+
+  ThreeDPoint listenerFrontNorm(listenerFront);
+  listenerFrontNorm.Normalize();
+
+  ThreeDPoint up = listenerRightNorm.CrossProduct(listenerFrontNorm);
+
+  double upProjection = sourceListener.DotProduct(up);
+
+  ThreeDPoint projectedSource = sourceListener - up * upProjection;
+  projectedSource.Normalize();
+
+  // Actually compute the angle, and convert to degrees
+  double projection = projectedSource.DotProduct(listenerRightNorm);
+  aAzimuth = 180 * acos(projection) / M_PI;
+
+  // Compute whether the source is in front or behind the listener.
+  double frontBack = projectedSource.DotProduct(listenerFrontNorm);
+  if (frontBack < 0) {
+    aAzimuth = 360 - aAzimuth;
+  }
+  // Rotate the azimuth so it is relative to the listener front vector instead
+  // of the right vector.
+  if ((aAzimuth >= 0) && (aAzimuth <= 270)) {
+    aAzimuth = 90 - aAzimuth;
+  } else {
+    aAzimuth = 450 - aAzimuth;
+  }
+
+  aElevation = 90 - 180 * acos(sourceListener.DotProduct(up)) / M_PI;
+
+  if (aElevation > 90) {
+    aElevation = 180 - aElevation;
+  } else if (aElevation < -90) {
+    aElevation = -180 - aElevation;
+  }
+}
+
 }
 }
 

content/media/webaudio/ThreeDPoint.cpp

@@ -0,0 +1,40 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * Other similar methods can be added if needed.
+ */
+
+#include "ThreeDPoint.h"
+
+namespace mozilla {
+
+namespace dom {
+
+ThreeDPoint operator-(const ThreeDPoint& lhs, const ThreeDPoint& rhs)
+{
+  return ThreeDPoint(lhs.x - rhs.x, lhs.y - rhs.y, lhs.z - rhs.z);
+}
+
+ThreeDPoint operator*(const ThreeDPoint& lhs, const ThreeDPoint& rhs)
+{
+  return ThreeDPoint(lhs.x * rhs.x, lhs.y * rhs.y, lhs.z * rhs.z);
+}
+
+ThreeDPoint operator*(const ThreeDPoint& lhs, const double rhs)
+{
+  return ThreeDPoint(lhs.x * rhs, lhs.y * rhs, lhs.z * rhs);
+}
+
+bool operator==(const ThreeDPoint& lhs, const ThreeDPoint& rhs)
+{
+  return lhs.x == rhs.x &&
+         lhs.y == rhs.y &&
+         lhs.z == rhs.z;
+}
+
+}
+}

content/media/webaudio/ThreeDPoint.h

@@ -7,6 +7,9 @@
 #ifndef ThreeDPoint_h_
 #define ThreeDPoint_h_
 
+#include <cmath>
+#include <cstdio>
+
 namespace mozilla {
 
 namespace dom {
@@ -25,9 +28,48 @@ struct ThreeDPoint {
   {
   }
 
+  double Magnitude() const
+  {
+    return sqrt(x * x + y * y + z * z);
+  }
+
+  void Normalize()
+  {
+    double invDistance = 1 / Magnitude();
+    x *= invDistance;
+    y *= invDistance;
+    z *= invDistance;
+  }
+
+  ThreeDPoint CrossProduct(const ThreeDPoint& rhs) const
+  {
+    return ThreeDPoint(y * rhs.z - z * rhs.y,
+                       z * rhs.x - x * rhs.z,
+                       x * rhs.y - y * rhs.x);
+  }
+
+  double DotProduct(const ThreeDPoint& rhs)
+  {
+    return x * rhs.x + y * rhs.y + z * rhs.z;
+  }
+
+  double Distance(const ThreeDPoint& rhs)
+  {
+    return sqrt(hypot(rhs.x, x) + hypot(rhs.y, y) + hypot(rhs.z, z));
+  }
+
+  bool IsZero() const
+  {
+    return x == 0 && y == 0 && z == 0;
+  }
   double x, y, z;
 };
 
+ThreeDPoint operator-(const ThreeDPoint& lhs, const ThreeDPoint& rhs);
+ThreeDPoint operator*(const ThreeDPoint& lhs, const ThreeDPoint& rhs);
+ThreeDPoint operator*(const ThreeDPoint& lhs, const double rhs);
+bool operator==(const ThreeDPoint& lhs, const ThreeDPoint& rhs);
+
 }
 }