Bug 1789846 - Using Stride and Height values from MediaCodec instead of assuming stride and height values. r=jolin,geckoview-reviewers,owlish

Differential Revision: https://phabricator.services.mozilla.com/D159130

dom/media/platforms/android/AndroidDataEncoder.cpp

@@ -170,14 +170,21 @@ RefPtr<MediaDataEncoder::EncodePromise> AndroidDataEncoder<ConfigType>::Encode(
 }
 
 static jni::ByteBuffer::LocalRef ConvertI420ToNV12Buffer(
-    RefPtr<const VideoData> aSample, RefPtr<MediaByteBuffer>& aYUVBuffer) {
+    RefPtr<const VideoData> aSample, RefPtr<MediaByteBuffer>& aYUVBuffer,
+    int aStride, int aYPlaneHeight) {
   const layers::PlanarYCbCrImage* image = aSample->mImage->AsPlanarYCbCrImage();
   MOZ_ASSERT(image);
   const layers::PlanarYCbCrData* yuv = image->GetData();
   auto ySize = yuv->YDataSize();
   auto cbcrSize = yuv->CbCrDataSize();
-  size_t yLength = yuv->mYStride * ySize.height;
-  size_t length = yLength + (yuv->mCbCrStride * cbcrSize.height * 2);
+  // If we have a stride or height passed in from the Codec we need to use
+  // those.
+  auto yStride = aStride != 0 ? aStride : yuv->mYStride;
+  auto height = aYPlaneHeight != 0 ? aYPlaneHeight : ySize.height;
+  size_t yLength = yStride * height;
+  size_t length =
+      yLength + yStride * (cbcrSize.height - 1) + cbcrSize.width * 2;
+
   if (!aYUVBuffer || aYUVBuffer->Capacity() < length) {
     aYUVBuffer = MakeRefPtr<MediaByteBuffer>(length);
     aYUVBuffer->SetLength(length);
@@ -187,9 +194,9 @@ static jni::ByteBuffer::LocalRef ConvertI420ToNV12Buffer(
 
   if (libyuv::I420ToNV12(yuv->mYChannel, yuv->mYStride, yuv->mCbChannel,
                          yuv->mCbCrStride, yuv->mCrChannel, yuv->mCbCrStride,
-                         aYUVBuffer->Elements(), yuv->mYStride,
-                         aYUVBuffer->Elements() + yLength, yuv->mCbCrStride * 2,
-                         ySize.width, ySize.height) != 0) {
+                         aYUVBuffer->Elements(), yStride,
+                         aYUVBuffer->Elements() + yLength, yStride, ySize.width,
+                         ySize.height) != 0) {
     return nullptr;
   }
 
@@ -206,8 +213,11 @@ AndroidDataEncoder<ConfigType>::ProcessEncode(RefPtr<const MediaData> aSample) {
   RefPtr<const VideoData> sample(aSample->As<const VideoData>());
   MOZ_ASSERT(sample);
 
-  jni::ByteBuffer::LocalRef buffer =
-      ConvertI420ToNV12Buffer(sample, mYUVBuffer);
+  // Bug 1789846: Check with the Encoder if MediaCodec has a stride or height
+  // value to use.
+  jni::ByteBuffer::LocalRef buffer = ConvertI420ToNV12Buffer(
+      sample, mYUVBuffer, mJavaEncoder->GetInputFormatStride(),
+      mJavaEncoder->GetInputFormatYPlaneHeight());
   if (!buffer) {
     return EncodePromise::CreateAndReject(NS_ERROR_ILLEGAL_INPUT, __func__);
   }

mobile/android/geckoview/src/main/aidl/org/mozilla/gecko/media/ICodec.aidl

@@ -14,7 +14,7 @@ import org.mozilla.gecko.media.SampleBuffer;
 
 interface ICodec {
     void setCallbacks(in ICodecCallbacks callbacks);
-    boolean configure(in FormatParam format, in GeckoSurface surface, in int flags, in String drmStubId);
+    boolean configure(inout FormatParam format, in GeckoSurface surface, in int flags, in String drmStubId);
     boolean isAdaptivePlaybackSupported();
     boolean isHardwareAccelerated();
     boolean isTunneledPlaybackSupported();

mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/AsyncCodec.java

@@ -46,6 +46,8 @@ public interface AsyncCodec {
 
   public abstract ByteBuffer getInputBuffer(int index);
 
+  public abstract MediaFormat getInputFormat();
+
   public abstract ByteBuffer getOutputBuffer(int index);
 
   public abstract void queueInputBuffer(

mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/Codec.java

@@ -427,6 +427,19 @@ import org.mozilla.gecko.gfx.GeckoSurface;
       }
       mIsHardwareAccelerated = !name.startsWith(SW_CODEC_PREFIX);
       mCodec = codec;
+      // Bug 1789846: Check if the Codec provides stride or height values to use.
+      if (flags == MediaCodec.CONFIGURE_FLAG_ENCODE && fmt.containsKey(MediaFormat.KEY_WIDTH)) {
+        final MediaFormat inputFormat = mCodec.getInputFormat();
+        if (inputFormat != null) {
+          if (inputFormat.containsKey(MediaFormat.KEY_STRIDE)) {
+            fmt.setInteger(MediaFormat.KEY_STRIDE, inputFormat.getInteger(MediaFormat.KEY_STRIDE));
+          }
+          if (inputFormat.containsKey(MediaFormat.KEY_SLICE_HEIGHT)) {
+            fmt.setInteger(
+                MediaFormat.KEY_SLICE_HEIGHT, inputFormat.getInteger(MediaFormat.KEY_SLICE_HEIGHT));
+          }
+        }
+      }
       mInputProcessor = new InputProcessor();
       final boolean renderToSurface = surface != null;
       mOutputProcessor = new OutputProcessor(renderToSurface);

mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/CodecProxy.java

@@ -132,6 +132,22 @@ public final class CodecProxy {
     }
   }
 
+  @WrapForJNI
+  public int GetInputFormatStride() {
+    if (mFormat.asFormat().containsKey(MediaFormat.KEY_STRIDE)) {
+      return mFormat.asFormat().getInteger(MediaFormat.KEY_STRIDE);
+    }
+    return 0;
+  }
+
+  @WrapForJNI
+  public int GetInputFormatYPlaneHeight() {
+    if (mFormat.asFormat().containsKey(MediaFormat.KEY_SLICE_HEIGHT)) {
+      return mFormat.asFormat().getInteger(MediaFormat.KEY_SLICE_HEIGHT);
+    }
+    return 0;
+  }
+
   @WrapForJNI
   public static CodecProxy create(
       final boolean isEncoder,

mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/FormatParam.java

@@ -24,6 +24,8 @@ import java.nio.ByteBuffer;
  *   <li>{@link MediaFormat#KEY_COLOR_FORMAT}
  *   <li>{@link MediaFormat#KEY_FRAME_RATE}
  *   <li>{@link MediaFormat#KEY_I_FRAME_INTERVAL}
+ *   <li>{@link MediaFormat#KEY_STRIDE}
+ *   <li>{@link MediaFormat#KEY_SLICE_HEIGHT}
  *   <li>"csd-0"
  *   <li>"csd-1"
  * </ul>
@@ -110,6 +112,12 @@ public final class FormatParam implements Parcelable {
       mFormat.setInteger(
           MediaFormat.KEY_I_FRAME_INTERVAL, bundle.getInt(MediaFormat.KEY_I_FRAME_INTERVAL));
     }
+    if (bundle.containsKey(MediaFormat.KEY_STRIDE)) {
+      mFormat.setInteger(MediaFormat.KEY_STRIDE, bundle.getInt(MediaFormat.KEY_STRIDE));
+    }
+    if (bundle.containsKey(MediaFormat.KEY_SLICE_HEIGHT)) {
+      mFormat.setInteger(MediaFormat.KEY_SLICE_HEIGHT, bundle.getInt(MediaFormat.KEY_SLICE_HEIGHT));
+    }
   }
 
   @Override
@@ -159,6 +167,12 @@ public final class FormatParam implements Parcelable {
       bundle.putInt(
           MediaFormat.KEY_I_FRAME_INTERVAL, mFormat.getInteger(MediaFormat.KEY_I_FRAME_INTERVAL));
     }
+    if (mFormat.containsKey(MediaFormat.KEY_STRIDE)) {
+      bundle.putInt(MediaFormat.KEY_STRIDE, mFormat.getInteger(MediaFormat.KEY_STRIDE));
+    }
+    if (mFormat.containsKey(MediaFormat.KEY_SLICE_HEIGHT)) {
+      bundle.putInt(MediaFormat.KEY_SLICE_HEIGHT, mFormat.getInteger(MediaFormat.KEY_SLICE_HEIGHT));
+    }
     return bundle;
   }
 }

mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/JellyBeanAsyncCodec.java

@@ -436,6 +436,11 @@ final class JellyBeanAsyncCodec implements AsyncCodec {
     return mOutputBuffers[index];
   }
 
+  @Override
+  public MediaFormat getInputFormat() {
+    return null;
+  }
+
   @Override
   public void flush() {
     assertCallbacks();

mobile/android/geckoview/src/main/java/org/mozilla/gecko/media/LollipopAsyncCodec.java

@@ -213,6 +213,11 @@ import org.mozilla.gecko.util.HardwareCodecCapabilityUtils;
     return mCodec.getOutputBuffer(index);
   }
 
+  @Override
+  public MediaFormat getInputFormat() {
+    return mCodec.getInputFormat();
+  }
+
   @Override
   public void queueInputBuffer(
       final int index,