Bug 1594141 - Handle failure to reconfigure video encoder in MediaEncoder. r=pehrsons

If the VP8 encoder fails to reconfigure in the middle of a stream it can
transition from a state where it was returning `true` for `IsInitialized` to
then returning false. This would trigger an assert in `GetEncodedData` in
`MediaEncoder`. This patch changes to instead handle this case by early
returning if we're in an error state, and ensuring we enter the error state if
a track encoder fails to reconfigure.

Update the TrackEncoder contract to differentiate the error used if encoding has
been canceled or completed, from other, more severe errors.

This also adjusts the logs in the vp8 track encoder to more clearly log if a
reconfigure of the encoder failed or not.

Differential Revision: https://phabricator.services.mozilla.com/D51956

--HG--
extra : moz-landing-system : lando

dom/media/encoder/MediaEncoder.cpp

@@ -678,6 +678,15 @@ nsresult MediaEncoder::GetEncodedData(
 
   MOZ_ASSERT(mEncoderThread->IsCurrentThreadIn());
   MOZ_ASSERT(mInitialized);
+
+  if (mError) {
+    // Bail if we're in an error state. This error state is likely caused by
+    // the underlying encoder(s) failing, so don't touch them.
+    LOG(LogLevel::Warning, ("GetEncodedData bailing early because we're in an "
+                            "error state (mError == true)"));
+    return NS_ERROR_ABORT;
+  }
+
   MOZ_ASSERT_IF(mAudioEncoder, mAudioEncoder->IsInitialized());
   MOZ_ASSERT_IF(mVideoEncoder, mVideoEncoder->IsInitialized());
 
@@ -712,8 +721,12 @@ nsresult MediaEncoder::GetEncodedData(
     nsTArray<RefPtr<EncodedFrame>> videoFrames;
     rv = mVideoEncoder->GetEncodedTrack(videoFrames);
     if (NS_FAILED(rv)) {
-      // Encoding might be canceled.
       LOG(LogLevel::Error, ("Failed to get encoded data from video encoder."));
+      // Encoding might be canceled. However, if our error isn't from
+      // cancellation, it indicates a more severe error, so set error state.
+      if (rv != NS_ERROR_DOM_MEDIA_CANCELED) {
+        SetError();
+      }
       return rv;
     }
     for (const RefPtr<EncodedFrame>& frame : videoFrames) {
@@ -728,8 +741,12 @@ nsresult MediaEncoder::GetEncodedData(
     nsTArray<RefPtr<EncodedFrame>> audioFrames;
     rv = mAudioEncoder->GetEncodedTrack(audioFrames);
     if (NS_FAILED(rv)) {
-      // Encoding might be canceled.
       LOG(LogLevel::Error, ("Failed to get encoded data from audio encoder."));
+      // Encoding might be canceled. However, if our error isn't from
+      // cancellation, it indicates a more severe error, so set error state.
+      if (rv != NS_ERROR_DOM_MEDIA_CANCELED) {
+        SetError();
+      }
       return rv;
     }
     for (const RefPtr<EncodedFrame>& frame : audioFrames) {

dom/media/encoder/OpusTrackEncoder.cpp

@@ -234,7 +234,7 @@ nsresult OpusTrackEncoder::GetEncodedTrack(
   MOZ_ASSERT(mInitialized || mCanceled);
 
   if (mCanceled || mEncodingComplete) {
-    return NS_ERROR_FAILURE;
+    return NS_ERROR_DOM_MEDIA_CANCELED;
   }
 
   if (!mInitialized) {

dom/media/encoder/TrackEncoder.h

@@ -80,6 +80,11 @@ class TrackEncoder {
   /**
    * Encodes raw segments. Result data is returned in aData, and called on the
    * worker thread.
+   *
+   * @returns NS_OK on success. NS_ERROR_DOM_MEDIA_CANCELED if encoding has
+   *          been canceled or completed, signaling that the encoder will not
+   *          emit more data. Other NS_ERROR_* codes will be returned for fatal
+   *          errors.
    */
   virtual nsresult GetEncodedTrack(nsTArray<RefPtr<EncodedFrame>>& aData) = 0;
 

dom/media/encoder/VP8TrackEncoder.cpp

@@ -339,8 +339,11 @@ nsresult VP8TrackEncoder::PrepareRawFrame(VideoChunk& aChunk) {
       Destroy();
       nsresult rv = Init(imgSize.width, imgSize.height, intrinsicSize.width,
                          intrinsicSize.height);
+      if (NS_FAILED(rv)) {
+        VP8LOG(LogLevel::Error, "Failed to recreate VP8 encoder.");
+        return rv;
+      }
       VP8LOG(LogLevel::Info, "Recreated VP8 encoder.");
-      NS_ENSURE_SUCCESS(rv, rv);
     }
   }
 
@@ -448,7 +451,7 @@ nsresult VP8TrackEncoder::GetEncodedTrack(
   MOZ_ASSERT(mInitialized || mCanceled);
 
   if (mCanceled || mEncodingComplete) {
-    return NS_ERROR_FAILURE;
+    return NS_ERROR_DOM_MEDIA_CANCELED;
   }
 
   if (!mInitialized) {