Bug 1014393 - Shift responsibility of adjusting packets with opus codec delay to MediaEncoder. r=pehrsons

Move the responsibility of adjusting opus frame timestamps to the MediaEncoder.
This was previously done by the EbmlComposer, but doing so in the MediaEncoder
means we can have greater control over handling of time codes and interleaving
of frames.

MozReview-Commit-ID: 2g9cy1IqOph

Differential Revision: https://phabricator.services.mozilla.com/D35385

--HG--
extra : moz-landing-system : lando

dom/media/encoder/MediaEncoder.cpp

@@ -917,6 +917,11 @@ nsresult MediaEncoder::EncodeData() {
       LOG(LogLevel::Error, ("Failed to get encoded data from audio encoder."));
       return rv;
     }
+    for (const RefPtr<EncodedFrame>& frame : mEncodedAudioFrames) {
+      if (frame->GetFrameType() == EncodedFrame::FrameType::OPUS_AUDIO_FRAME) {
+        frame->SetTimeStamp(frame->GetTimeStamp() + mAudioCodecDelay);
+      }
+    }
   }
 
   return NS_OK;
@@ -978,6 +983,17 @@ nsresult MediaEncoder::CopyMetadataToMuxer(TrackEncoder* aTrackEncoder) {
     return NS_ERROR_ABORT;
   }
 
+  // In the case of Opus we need to calculate the codec delay based on the
+  // pre-skip. For more information see:
+  // https://tools.ietf.org/html/rfc7845#section-4.2
+  if (meta->GetKind() == TrackMetadataBase::MetadataKind::METADATA_OPUS) {
+    // Calculate offset in microseconds
+    OpusMetadata* opusMeta = static_cast<OpusMetadata*>(meta.get());
+    mAudioCodecDelay = static_cast<uint64_t>(
+        LittleEndian::readUint16(opusMeta->mIdHeader.Elements() + 10) *
+        PR_USEC_PER_SEC / 48000);
+  }
+
   nsresult rv = mWriter->SetMetadata(meta);
   if (NS_FAILED(rv)) {
     LOG(LogLevel::Error, ("SetMetadata failed"));

dom/media/encoder/MediaEncoder.h

@@ -292,6 +292,10 @@ class MediaEncoder {
   // Video frames that have been encoded and are pending write to the muxer
   nsTArray<RefPtr<EncodedFrame>> mEncodedVideoFrames;
 
+  // How much each audio time stamp should be delayed in microseconds. Used to
+  // adjust for opus codec delay.
+  uint64_t mAudioCodecDelay = 0;
+
   TimeStamp mStartTime;
   nsString mMIMEType;
   bool mInitialized;

dom/media/webm/EbmlComposer.cpp

@@ -55,14 +55,15 @@ void EbmlComposer::GenerateHeader() {
           if (mCodecPrivateData.Length() > 0) {
             // Extract the pre-skip from mCodecPrivateData
             // then convert it to nanoseconds.
-            // Details in OpusTrackEncoder.cpp.
-            mCodecDelay = (uint64_t)LittleEndian::readUint16(
-                              mCodecPrivateData.Elements() + 10) *
-                          PR_NSEC_PER_SEC / 48000;
+            // For more details see
+            // https://tools.ietf.org/html/rfc7845#section-4.2
+            uint64_t codecDelay = (uint64_t)LittleEndian::readUint16(
+                                      mCodecPrivateData.Elements() + 10) *
+                                  PR_NSEC_PER_SEC / 48000;
             // Fixed 80ms, convert into nanoseconds.
             uint64_t seekPreRoll = 80 * PR_NSEC_PER_MSEC;
             writeAudioTrack(&ebml, 0x2, 0x0, "A_OPUS", mSampleFreq, mChannels,
-                            mCodecDelay, seekPreRoll,
+                            codecDelay, seekPreRoll,
                             mCodecPrivateData.Elements(),
                             mCodecPrivateData.Length());
           }
@@ -130,9 +131,6 @@ void EbmlComposer::WriteSimpleBlock(EncodedFrame* aFrame) {
 
   int64_t timeCode =
       aFrame->GetTimeStamp() / ((int)PR_USEC_PER_MSEC) - mClusterTimecode;
-  if (isOpus) {
-    timeCode += mCodecDelay / PR_NSEC_PER_MSEC;
-  }
 
   if (!mHasVideo && timeCode >= FLUSH_AUDIO_ONLY_AFTER_MS) {
     MOZ_ASSERT(mHasAudio);
@@ -163,9 +161,6 @@ void EbmlComposer::WriteSimpleBlock(EncodedFrame* aFrame) {
     // Can't under-/overflow now
     timeCode =
         aFrame->GetTimeStamp() / ((int)PR_USEC_PER_MSEC) - mClusterTimecode;
-    if (isOpus) {
-      timeCode += mCodecDelay / PR_NSEC_PER_MSEC;
-    }
 
     mWritingCluster = true;
   }

dom/media/webm/EbmlComposer.h

@@ -38,7 +38,8 @@ class EbmlComposer {
   /*
    * Insert media encoded buffer into muxer and it would be package
    * into SimpleBlock. If no cluster is opened, new cluster will start for
-   * writing.
+   * writing. Frames passed to this function should already have any codec delay
+   * applied.
    */
   void WriteSimpleBlock(EncodedFrame* aFrame);
   /*
@@ -68,8 +69,6 @@ class EbmlComposer {
   uint64_t mClusterLengthLoc = 0;
   // Audio codec specific header data.
   nsTArray<uint8_t> mCodecPrivateData;
-  // Codec delay in nanoseconds.
-  uint64_t mCodecDelay = 0;
 
   // The timecode of the cluster.
   uint64_t mClusterTimecode = 0;