Bug 1215115 - part2: Mux opus into webm, remove bitdepth. r=rillian

MozReview-Commit-ID: 7D74vONbkFP

--HG--
extra : transplant_source : 2%AA%C8%F34%1E%CF%3C%9B%88%AC%AD%9F-%DC%7C%FE%B1m%5D

dom/media/encoder/OpusTrackEncoder.cpp

@@ -129,6 +129,7 @@ OpusTrackEncoder::OpusTrackEncoder()
   , mEncoder(nullptr)
   , mLookahead(0)
   , mResampler(nullptr)
+  , mOutputTimeStamp(0)
 {
 }
 
@@ -229,6 +230,8 @@ OpusTrackEncoder::GetMetadata()
   }
 
   RefPtr<OpusMetadata> meta = new OpusMetadata();
+  meta->mChannels = mChannels;
+  meta->mSamplingFrequency = mSamplingRate;
 
   mLookahead = 0;
   int error = opus_encoder_ctl(mEncoder, OPUS_GET_LOOKAHEAD(&mLookahead));
@@ -437,6 +440,9 @@ OpusTrackEncoder::GetEncodedTrack(EncodedFrameContainer& aData)
   }
 
   audiodata->SwapInFrameData(frameData);
+  mOutputTimeStamp += FramesToUsecs(GetPacketDuration(), kOpusSamplingRate).value();
+  audiodata->SetTimeStamp(mOutputTimeStamp);
+  LOG("[Opus] mOutputTimeStamp %lld.",mOutputTimeStamp);
   aData.AppendEncodedFrame(audiodata);
   return result >= 0 ? NS_OK : NS_ERROR_FAILURE;
 }

dom/media/encoder/OpusTrackEncoder.h

@@ -22,7 +22,8 @@ public:
   nsTArray<uint8_t> mIdHeader;
   // The Comment Header of OggOpus.
   nsTArray<uint8_t> mCommentHeader;
-
+  int32_t mChannels;
+  float mSamplingFrequency;
   MetadataKind GetKind() const override { return METADATA_OPUS; }
 };
 
@@ -80,6 +81,9 @@ private:
    * They will be prepended to the resampled frames next encoding cycle.
    */
   nsTArray<AudioDataValue> mResampledLeftover;
+
+  // TimeStamp in microseconds.
+  uint64_t mOutputTimeStamp;
 };
 
 } // namespace mozilla

dom/media/webm/EbmlComposer.cpp

@@ -5,6 +5,7 @@
 
 #include "EbmlComposer.h"
 #include "mozilla/UniquePtr.h"
+#include "mozilla/Endian.h"
 #include "libmkv/EbmlIDs.h"
 #include "libmkv/EbmlWriter.h"
 #include "libmkv/WebMElement.h"
@@ -48,8 +49,17 @@ void EbmlComposer::GenerateHeader()
           }
           // Audio
           if (mCodecPrivateData.Length() > 0) {
-            writeAudioTrack(&ebml, 0x2, 0x0, "A_VORBIS", mSampleFreq,
-                            mChannels, mCodecPrivateData.Elements(),
+            // Extract the pre-skip from mCodecPrivateData
+            // then convert it to nanoseconds.
+            // Details in OpusTrackEncoder.cpp.
+            mCodecDelay =
+              (uint64_t)LittleEndian::readUint16(mCodecPrivateData.Elements() + 10)
+              * PR_NSEC_PER_SEC / 48000;
+            // Fixed 80ms, convert into nanoseconds.
+            uint64_t seekPreRoll = 80 * PR_NSEC_PER_MSEC;
+            writeAudioTrack(&ebml, 0x2, 0x0, "A_OPUS", mSampleFreq,
+                            mChannels, mCodecDelay, seekPreRoll,
+                            mCodecPrivateData.Elements(),
                             mCodecPrivateData.Length());
           }
         }
@@ -135,9 +145,12 @@ EbmlComposer::WriteSimpleBlock(EncodedFrame* aFrame)
     mFlushState |= FLUSH_CLUSTER;
   }
 
-  bool isVorbis = (frameType == EncodedFrame::FrameType::VORBIS_AUDIO_FRAME);
+  bool isOpus = (frameType == EncodedFrame::FrameType::OPUS_AUDIO_FRAME);
   short timeCode = aFrame->GetTimeStamp() / PR_USEC_PER_MSEC - mClusterTimecode;
-  writeSimpleBlock(&ebml, isVorbis ? 0x2 : 0x1, timeCode, isVP8IFrame,
+  if (isOpus) {
+    timeCode += mCodecDelay / PR_NSEC_PER_MSEC;
+  }
+  writeSimpleBlock(&ebml, isOpus ? 0x2 : 0x1, timeCode, isVP8IFrame,
                    0, 0, (unsigned char*)aFrame->GetFrameData().Elements(),
                    aFrame->GetFrameData().Length());
   MOZ_ASSERT(ebml.offset <= DEFAULT_HEADER_SIZE +
@@ -164,14 +177,11 @@ EbmlComposer::SetVideoConfig(uint32_t aWidth, uint32_t aHeight,
 }
 
 void
-EbmlComposer::SetAudioConfig(uint32_t aSampleFreq, uint32_t aChannels,
-                             uint32_t aBitDepth)
+EbmlComposer::SetAudioConfig(uint32_t aSampleFreq, uint32_t aChannels)
 {
   MOZ_ASSERT(aSampleFreq > 0, "SampleFreq should > 0");
-  MOZ_ASSERT(aBitDepth > 0, "BitDepth should > 0");
   MOZ_ASSERT(aChannels > 0, "Channels should > 0");
   mSampleFreq = aSampleFreq;
-  mBitDepth = aBitDepth;
   mChannels = aChannels;
 }
 
@@ -199,12 +209,12 @@ EbmlComposer::EbmlComposer()
   : mFlushState(FLUSH_NONE)
   , mClusterHeaderIndex(0)
   , mClusterLengthLoc(0)
+  , mCodecDelay(0)
   , mClusterTimecode(0)
   , mWidth(0)
   , mHeight(0)
   , mFrameRate(0)
   , mSampleFreq(0)
-  , mBitDepth(0)
   , mChannels(0)
 {}
 

dom/media/webm/EbmlComposer.h

@@ -22,8 +22,7 @@ public:
   void SetVideoConfig(uint32_t aWidth, uint32_t aHeight, uint32_t aDisplayWidth,
                       uint32_t aDisplayHeight, float aFrameRate);
 
-  void SetAudioConfig(uint32_t aSampleFreq, uint32_t aChannels,
-                      uint32_t bitDepth);
+  void SetAudioConfig(uint32_t aSampleFreq, uint32_t aChannels);
   /*
    * Set the CodecPrivateData for writing in header.
    */
@@ -68,6 +67,8 @@ private:
   uint64_t mClusterLengthLoc;
   // Audio codec specific header data.
   nsTArray<uint8_t> mCodecPrivateData;
+  // Codec delay in nanoseconds.
+  uint64_t mCodecDelay;
 
   // The timecode of the cluster.
   uint64_t mClusterTimecode;
@@ -80,7 +81,6 @@ private:
   float mFrameRate;
   // Audio configuration
   float mSampleFreq;
-  int mBitDepth;
   int mChannels;
 };
 

dom/media/webm/WebMWriter.cpp

@@ -6,6 +6,7 @@
 #include "WebMWriter.h"
 #include "EbmlComposer.h"
 #include "GeckoProfiler.h"
+#include "OpusTrackEncoder.h"
 
 namespace mozilla {
 
@@ -64,11 +65,19 @@ WebMWriter::SetMetadata(TrackMetadataBase* aMetadata)
   if (aMetadata->GetKind() == TrackMetadataBase::METADATA_VORBIS) {
     VorbisMetadata* meta = static_cast<VorbisMetadata*>(aMetadata);
     MOZ_ASSERT(meta, "Cannot find vorbis encoder metadata");
-    mEbmlComposer->SetAudioConfig(meta->mSamplingFrequency, meta->mChannels, meta->mBitDepth);
+    mEbmlComposer->SetAudioConfig(meta->mSamplingFrequency, meta->mChannels);
     mEbmlComposer->SetAudioCodecPrivateData(meta->mData);
     mMetadataRequiredFlag = mMetadataRequiredFlag & ~ContainerWriter::CREATE_AUDIO_TRACK;
   }
 
+  if (aMetadata->GetKind() == TrackMetadataBase::METADATA_OPUS) {
+    OpusMetadata* meta = static_cast<OpusMetadata*>(aMetadata);
+    MOZ_ASSERT(meta, "Cannot find Opus encoder metadata");
+    mEbmlComposer->SetAudioConfig(meta->mSamplingFrequency, meta->mChannels);
+    mEbmlComposer->SetAudioCodecPrivateData(meta->mIdHeader);
+    mMetadataRequiredFlag = mMetadataRequiredFlag & ~ContainerWriter::CREATE_AUDIO_TRACK;
+  }
+
   if (!mMetadataRequiredFlag) {
     mEbmlComposer->GenerateHeader();
   }

dom/media/webm/WebMWriter.h

@@ -18,7 +18,6 @@ class VorbisMetadata : public TrackMetadataBase
 public:
   nsTArray<uint8_t> mData;
   int32_t mChannels;
-  int32_t mBitDepth;
   float mSamplingFrequency;
   MetadataKind GetKind() const override { return METADATA_VORBIS; }
 };

media/libmkv/EbmlIDs.h

@@ -106,6 +106,8 @@ enum mkv {
   CodecID = 0x86,
   CodecPrivate = 0x63A2,
   CodecName = 0x258688,
+  CodecDelay = 0x56AA,
+  SeekPreRoll = 0x56BB,
 /* AttachmentLink = 0x7446, */
 /* CodecSettings = 0x3A9697, */
 /* CodecInfoURL = 0x3B4040, */

media/libmkv/WebMElement.c

@@ -87,6 +87,7 @@ void writeVideoTrack(EbmlGlobal *glob, unsigned int trackNumber, int flagLacing,
 }
 void writeAudioTrack(EbmlGlobal *glob, unsigned int trackNumber, int flagLacing,
                      const char *codecId, double samplingFrequency, unsigned int channels,
+                     uint64_t codecDelay, uint64_t seekPreRoll,
                      unsigned char *private, unsigned long privateSize) {
   EbmlLoc start;
   UInt64 trackID;
@@ -95,6 +96,8 @@ void writeAudioTrack(EbmlGlobal *glob, unsigned int trackNumber, int flagLacing,
   trackID = generateTrackID(trackNumber);
   Ebml_SerializeUnsigned(glob, TrackUID, trackID);
   Ebml_SerializeUnsigned(glob, TrackType, 2); // audio is always 2
+  Ebml_SerializeUnsigned(glob, CodecDelay, codecDelay);
+  Ebml_SerializeUnsigned(glob, SeekPreRoll, seekPreRoll);
   // I am using defaults for thesed required fields
   /*  Ebml_SerializeUnsigned(glob, FlagEnabled, 1);
       Ebml_SerializeUnsigned(glob, FlagDefault, 1);
@@ -103,7 +106,7 @@ void writeAudioTrack(EbmlGlobal *glob, unsigned int trackNumber, int flagLacing,
   Ebml_SerializeString(glob, CodecID, codecId);
   Ebml_SerializeData(glob, CodecPrivate, private, privateSize);
 
-  Ebml_SerializeString(glob, CodecName, "VORBIS");  // fixed for now
+  Ebml_SerializeString(glob, CodecName, "OPUS");  // fixed for now
   {
     EbmlLoc AudioStart;
     Ebml_StartSubElement(glob, &AudioStart, Audio);

media/libmkv/WebMElement.h

@@ -25,6 +25,7 @@ void writeVideoTrack(EbmlGlobal *ebml, unsigned int trackNumber, int flagLacing,
                      double frameRate);
 void writeAudioTrack(EbmlGlobal *glob, unsigned int trackNumber, int flagLacing,
                      const char *codecId, double samplingFrequency, unsigned int channels,
+                     uint64_t codecDelay, uint64_t seekPreRoll,
                      unsigned char *private_, unsigned long privateSize);
 
 void writeSimpleBlock(EbmlGlobal *ebml, unsigned char trackNumber, short timeCode,