Bug 748144 - Support multichannel Opus files. r=rillian

The Opus audio format supports multichannel (surround) audio, but our initial implementation used a simpler API which only supported mono and stereo output.

To handle these files gracefully, this patch uses the multichannel api and downmixes the output, if possible, to stereo, since we don't currently support surround sound playback.

content/media/ogg/nsOggCodecState.cpp

@@ -770,6 +770,7 @@ nsOpusState::nsOpusState(ogg_page* aBosPage) :
 #endif
   mChannelMapping(0),
   mStreams(0),
+  mCoupledStreams(0),
   mDecoder(NULL),
   mSkip(0),
   mPrevPacketGranulepos(0),
@@ -783,7 +784,7 @@ nsOpusState::~nsOpusState() {
   Reset();
 
   if (mDecoder) {
-    opus_decoder_destroy(mDecoder);
+    opus_multistream_decoder_destroy(mDecoder);
     mDecoder = NULL;
   }
 }
@@ -799,7 +800,7 @@ nsresult nsOpusState::Reset(bool aStart)
 
   if (mActive && mDecoder) {
     // Reset the decoder.
-    opus_decoder_ctl(mDecoder, OPUS_RESET_STATE);
+    opus_multistream_decoder_ctl(mDecoder, OPUS_RESET_STATE);
     // Let the seek logic handle pre-roll if we're not seeking to the start.
     mSkip = aStart ? mPreSkip : 0;
     // This lets us distinguish the first page being the last page vs. just
@@ -827,7 +828,13 @@ bool nsOpusState::Init(void)
 
   NS_ASSERTION(mDecoder == NULL, "leaking OpusDecoder");
 
-  mDecoder = opus_decoder_create(mRate, mChannels, &error);
+  mDecoder = opus_multistream_decoder_create(mRate,
+                                             mChannels,
+                                             mStreams,
+                                             mCoupledStreams,
+                                             mMappingTable,
+                                             &error);
+
   mSkip = mPreSkip;
 
   LOG(PR_LOG_DEBUG, ("Opus decoder init, to skip %d", mSkip));
@@ -855,7 +862,17 @@ bool nsOpusState::DecodeHeader(ogg_packet* aPacket)
         return false;
       }
 
-      mChannels= aPacket->packet[9];
+      mChannels = aPacket->packet[9];
+      if (mChannels<1) {
+        LOG(PR_LOG_DEBUG, ("Invalid Opus file: Number of channels %d", mChannels));
+        return false;
+      }
+#ifndef MOZ_SAMPLE_TYPE_FLOAT32
+      // Downmixing more than 2 channels it is not supported for integer
+      // output samples. It is only supported for float output.
+      if (mChannels>2)
+        return false;
+#endif
       mPreSkip = LEUint16(aPacket->packet + 10);
       mNominalRate = LEUint32(aPacket->packet + 12);
       double gain_dB = LEInt16(aPacket->packet + 16) / 256.0;
@@ -869,8 +886,15 @@ bool nsOpusState::DecodeHeader(ogg_packet* aPacket)
 
       if (mChannelMapping == 0) {
         mStreams = 1;
-      } else if (aPacket->bytes > 19) {
+        mCoupledStreams = mChannels - 1;
+        mMappingTable[0] = 0;
+        mMappingTable[1] = 1;
+      } else if (aPacket->bytes>20+mChannels) {
         mStreams = aPacket->packet[19];
+        mCoupledStreams = aPacket->packet[20];
+        int i;
+        for (i=0; i<mChannels; i++)
+          mMappingTable[i] = aPacket->packet[21+i];
       } else {
         LOG(PR_LOG_DEBUG, ("Invalid Opus file: channel mapping %d,"
                            " but no channel mapping table", mChannelMapping));

content/media/ogg/nsOggCodecState.h

@@ -15,6 +15,9 @@
 #endif
 #ifdef MOZ_OPUS
 #include <opus/opus.h>
+extern "C" {
+#include "opus/opus_multistream.h"
+}
 // For MOZ_SAMPLE_TYPE_*
 #include "nsBuiltinDecoderStateMachine.h"
 #include "nsBuiltinDecoderReader.h"
@@ -326,8 +329,11 @@ public:
 #endif
   int mChannelMapping; // Channel mapping family.
   int mStreams;     // Number of packed streams in each packet.
+  int mCoupledStreams; // Number of packed coupled streams in each packet.
+  unsigned char mMappingTable[255]; // Channel mapping table.
+
+  OpusMSDecoder *mDecoder;
 
-  OpusDecoder *mDecoder;
   int mSkip;        // Number of samples left to trim before playback.
   // Granule position (end sample) of the last decoded Opus packet. This is
   // used to calculate the amount we should trim from the last packet.

content/media/ogg/nsOggReader.cpp

@@ -11,6 +11,9 @@
 #include "theora/theoradec.h"
 #ifdef MOZ_OPUS
 #include "opus/opus.h"
+extern "C" {
+#include "opus/opus_multistream.h"
+}
 #endif
 #include "nsTimeRanges.h"
 #include "mozilla/TimeStamp.h"
@@ -321,7 +324,7 @@ nsresult nsOggReader::ReadMetadata(nsVideoInfo* aInfo,
   if (mOpusState && ReadHeaders(mOpusState)) {
     mInfo.mHasAudio = true;
     mInfo.mAudioRate = mOpusState->mRate;
-    mInfo.mAudioChannels = mOpusState->mChannels;
+    mInfo.mAudioChannels = mOpusState->mChannels > 2 ? 2 : mOpusState->mChannels;
     mOpusSerial = mOpusState->mSerial;
     mOpusPreSkip = mOpusState->mPreSkip;
   }
@@ -423,9 +426,12 @@ nsresult nsOggReader::DecodeOpus(ogg_packet* aPacket) {
   NS_ASSERTION(aPacket->granulepos != -1, "Must know opus granulepos!");
 
   // Maximum value is 63*2880.
-  PRInt32 frames = opus_decoder_get_nb_samples(mOpusState->mDecoder,
-                                               aPacket->packet,
+  PRInt32 frames_number = opus_packet_get_nb_frames(aPacket->packet,
                                                     aPacket->bytes);
+  PRInt32 samples = opus_packet_get_samples_per_frame(aPacket->packet,
+                                                      (opus_int32) mOpusState->mRate);
+  PRInt32 frames = frames_number*samples;
+
   if (frames <= 0)
     return NS_ERROR_FAILURE;
   PRUint32 channels = mOpusState->mChannels;
@@ -433,11 +439,11 @@ nsresult nsOggReader::DecodeOpus(ogg_packet* aPacket) {
 
   // Decode to the appropriate sample type.
 #ifdef MOZ_SAMPLE_TYPE_FLOAT32
-  int ret = opus_decode_float(mOpusState->mDecoder,
+  int ret = opus_multistream_decode_float(mOpusState->mDecoder,
                                           aPacket->packet, aPacket->bytes,
                                           buffer, frames, false);
 #else
-  int ret = opus_decode(mOpusState->mDecoder,
+  int ret = opus_multistream_decode(mOpusState->mDecoder,
                                     aPacket->packet, aPacket->bytes,
                                     buffer, frames, false);
 #endif
@@ -504,6 +510,45 @@ nsresult nsOggReader::DecodeOpus(ogg_packet* aPacket) {
   }
 #endif
 
+  // More than 2 decoded channels must be downmixed to stereo.
+  if (channels > 2) {
+    // Opus doesn't provide a channel mapping for more than 8 channels,
+    // so we can't downmix more than that.
+    if (channels > 8)
+      return NS_ERROR_FAILURE;
+
+#ifdef MOZ_SAMPLE_TYPE_FLOAT32
+    PRUint32 out_channels;
+    out_channels = 2;
+
+    // dBuffer stores the downmixed sample data.
+    nsAutoArrayPtr<AudioDataValue> dBuffer(new AudioDataValue[frames * out_channels]);
+    // Downmix matrix for channels up to 8, normalized to 2.0.
+    static const float dmatrix[6][8][2]= {
+        /*3*/{ {0.5858f,0}, {0.4142f,0.4142f}, {0,0.5858f}},
+        /*4*/{ {0.4226f,0}, {0,0.4226f}, {0.366f,0.2114f}, {0.2114f,0.366f}},
+        /*5*/{ {0.651f,0}, {0.46f,0.46f}, {0,0.651f}, {0.5636f,0.3254f}, {0.3254f,0.5636f}},
+        /*6*/{ {0.529f,0}, {0.3741f,0.3741f}, {0,0.529f}, {0.4582f,0.2645f}, {0.2645f,0.4582f}, {0.3741f,0.3741f}},
+        /*7*/{ {0.4553f,0}, {0.322f,0.322f}, {0,4553}, {0.3943f,0.2277f}, {0.2277f,0.3943f}, {0.2788f,0.2788f}, {0.322f,0.322f}},
+        /*8*/{ {0.3886f,0}, {0.2748f,0.2748f}, {0,0.3886f}, {0.3366f,0.1943f}, {0.1943f,0.3366f}, {0.3366f,0.1943f}, {0.1943f,0.3366f}, {0.2748f,0.2748f}},
+    };
+    for (PRInt32 i = 0; i < frames; i++) {
+      float sampL = 0.0;
+      float sampR = 0.0;
+      for (PRUint32 j = 0; j < channels; j++) {
+        sampL+=buffer[i*channels+j]*dmatrix[channels-3][j][0];
+        sampR+=buffer[i*channels+j]*dmatrix[channels-3][j][1];
+      }
+      dBuffer[i*out_channels]=sampL;
+      dBuffer[i*out_channels+1]=sampR;
+    }
+    channels = out_channels;
+    buffer = dBuffer;
+#else
+  return NS_ERROR_FAILURE;
+#endif
+  }
+
   LOG(PR_LOG_DEBUG, ("Opus decoder pushing %d frames", frames));
   PRInt64 startTime = mOpusState->Time(startFrame);
   PRInt64 endTime = mOpusState->Time(endFrame);

layout/media/symbols.def.in

@@ -134,6 +134,11 @@ opus_decoder_ctl
 opus_decoder_get_nb_samples
 opus_decode
 opus_decode_float
+opus_multistream_decoder_create
+opus_multistream_decoder_ctl
+opus_multistream_decoder_destroy
+opus_multistream_decode_float
+opus_multistream_decode
 opus_packet_get_nb_frames
 opus_packet_get_samples_per_frame
 #endif