Bug 960243 - Try to fetch m frames such that there will be n frames after resampling which will fit into an Opus packet duration perfectly. r=rillian

content/media/encoder/OpusTrackEncoder.cpp

@@ -244,16 +244,43 @@ OpusTrackEncoder::GetMetadata()
 nsresult
 OpusTrackEncoder::GetEncodedTrack(EncodedFrameContainer& aData)
 {
+  {
+    ReentrantMonitorAutoEnter mon(mReentrantMonitor);
+    // Wait until initialized or cancelled.
+    while (!mCanceled && !mInitialized) {
+      mReentrantMonitor.Wait();
+    }
+    if (mCanceled || mEncodingComplete) {
+      return NS_ERROR_FAILURE;
+    }
+  }
+
+  // calculation below depends on the truth that mInitialized is true.
+  MOZ_ASSERT(mInitialized);
+
+  // re-sampled frames left last time which didn't fit into an Opus packet duration.
+  const int framesLeft = mResampledLeftover.Length() / mChannels;
+  // When framesLeft is 0, (GetPacketDuration() - framesLeft) is a multiple
+  // of kOpusSamplingRate. There is not precision loss in the integer division
+  // in computing framesToFetch. If frameLeft > 0, we need to add 1 to
+  // framesToFetch to ensure there will be at least n frames after re-sampling.
+  const int frameRoundUp = framesLeft ? 1 : 0;
+
+  MOZ_ASSERT(GetPacketDuration() >= framesLeft);
+  // Try to fetch m frames such that there will be n frames
+  // where (n + frameLeft) >= GetPacketDuration() after re-sampling.
+  const int framesToFetch = !mResampler ? GetPacketDuration()
+    : (GetPacketDuration() - framesLeft) * mSamplingRate / kOpusSamplingRate
+      + frameRoundUp;
   {
     // Move all the samples from mRawSegment to mSourceSegment. We only hold
     // the monitor in this block.
     ReentrantMonitorAutoEnter mon(mReentrantMonitor);
 
-    // Wait if mEncoder is not initialized, or when not enough raw data, but is
-    // not the end of stream nor is being canceled.
-    while (!mCanceled && (!mInitialized || (mRawSegment.GetDuration() +
-           mSourceSegment.GetDuration() < GetPacketDuration() &&
-           !mEndOfStream))) {
+    // Wait until enough raw data, end of stream or cancelled.
+    while (!mCanceled && mRawSegment.GetDuration() +
+        mSourceSegment.GetDuration() < framesToFetch &&
+        !mEndOfStream) {
       mReentrantMonitor.Wait();
     }
 
@@ -276,13 +303,14 @@ OpusTrackEncoder::GetEncodedTrack(EncodedFrameContainer& aData)
   pcm.SetLength(GetPacketDuration() * mChannels);
   AudioSegment::ChunkIterator iter(mSourceSegment);
   int frameCopied = 0;
-  while (!iter.IsEnded() && frameCopied < GetPacketDuration()) {
+
+  while (!iter.IsEnded() && frameCopied < framesToFetch) {
     AudioChunk chunk = *iter;
 
     // Chunk to the required frame size.
     int frameToCopy = chunk.GetDuration();
-    if (frameCopied + frameToCopy > GetPacketDuration()) {
-      frameToCopy = GetPacketDuration() - frameCopied;
+    if (frameCopied + frameToCopy > framesToFetch) {
+      frameToCopy = framesToFetch - frameCopied;
     }
 
     if (!chunk.IsNull()) {
@@ -300,6 +328,7 @@ OpusTrackEncoder::GetEncodedTrack(EncodedFrameContainer& aData)
 
   nsRefPtr<EncodedFrame> audiodata = new EncodedFrame();
   audiodata->SetFrameType(EncodedFrame::AUDIO_FRAME);
+  int framesInPCM = frameCopied;
   if (mResampler) {
     nsAutoTArray<AudioDataValue, 9600> resamplingDest;
     // We want to consume all the input data, so we slightly oversize the
@@ -322,9 +351,25 @@ OpusTrackEncoder::GetEncodedTrack(EncodedFrameContainer& aData)
                                                           out, &outframes);
 #endif
 
-    pcm = resamplingDest;
+    MOZ_ASSERT(pcm.Length() >= mResampledLeftover.Length());
+    PodCopy(pcm.Elements(), mResampledLeftover.Elements(),
+        mResampledLeftover.Length());
+
+    uint32_t outframesToCopy = std::min(outframes,
+        static_cast<uint32_t>(GetPacketDuration() - framesLeft));
+
+    MOZ_ASSERT(pcm.Length() - mResampledLeftover.Length() >=
+        outframesToCopy * mChannels);
+    PodCopy(pcm.Elements() + mResampledLeftover.Length(),
+        resamplingDest.Elements(), outframesToCopy * mChannels);
+    int frameLeftover = outframes - outframesToCopy;
+    mResampledLeftover.SetLength(frameLeftover * mChannels);
+    PodCopy(mResampledLeftover.Elements(),
+        resamplingDest.Elements() + outframesToCopy * mChannels,
+        mResampledLeftover.Length());
     // This is always at 48000Hz.
-    audiodata->SetDuration(outframes);
+    framesInPCM = framesLeft + outframesToCopy;
+    audiodata->SetDuration(framesInPCM);
   } else {
     // The ogg time stamping and pre-skip is always timed at 48000.
     audiodata->SetDuration(frameCopied * (kOpusSamplingRate / mSamplingRate));
@@ -342,11 +387,13 @@ OpusTrackEncoder::GetEncodedTrack(EncodedFrameContainer& aData)
     LOG("[Opus] Done encoding.");
   }
 
+  MOZ_ASSERT(mEndOfStream || framesInPCM == GetPacketDuration());
+
   // Append null data to pcm buffer if the leftover data is not enough for
   // opus encoder.
-  if (frameCopied < GetPacketDuration() && mEndOfStream) {
-    memset(pcm.Elements() + frameCopied * mChannels, 0,
-           (GetPacketDuration()-frameCopied)*mChannels*sizeof(AudioDataValue));
+  if (framesInPCM < GetPacketDuration() && mEndOfStream) {
+    PodZero(pcm.Elements() + framesInPCM * mChannels,
+        (GetPacketDuration() - framesInPCM) * mChannels);
   }
   nsTArray<uint8_t> frameData;
   // Encode the data with Opus Encoder.
@@ -372,6 +419,7 @@ OpusTrackEncoder::GetEncodedTrack(EncodedFrameContainer& aData)
       speex_resampler_destroy(mResampler);
       mResampler = nullptr;
     }
+    mResampledLeftover.SetLength(0);
   }
 
   audiodata->SwapInFrameData(frameData);

content/media/encoder/OpusTrackEncoder.h

@@ -74,6 +74,12 @@ private:
    * resampled.
    */
   SpeexResamplerState* mResampler;
+
+  /**
+   * Store the resampled frames that don't fit into an Opus packet duration.
+   * They will be prepended to the resampled frames next encoding cycle.
+   */
+  nsTArray<AudioDataValue> mResampledLeftover;
 };
 
 }