Bug 1299068 - part 5: release/render buffers when VideoData sent to compositor. r=snorp

MozReview-Commit-ID: JmEKLKlJnaL

--HG--
extra : rebase_source : 5177ed35206aad3423b9960ea5e9e59459540cde

dom/media/platforms/android/RemoteDataDecoder.cpp

@@ -114,6 +114,40 @@ struct SampleTime final
 class RemoteVideoDecoder final : public RemoteDataDecoder
 {
 public:
+  // Hold an output buffer and render it to the surface when the frame is sent to compositor, or
+  // release it if not presented.
+  class RenderOrReleaseOutput : public VideoData::Listener
+  {
+  public:
+    RenderOrReleaseOutput(java::CodecProxy::Param aCodec, java::Sample::Param aSample)
+      : mCodec(aCodec),
+        mSample(aSample)
+    {}
+
+    ~RenderOrReleaseOutput()
+    {
+      ReleaseOutput(false);
+    }
+
+    void OnSentToCompositor() override
+    {
+      ReleaseOutput(true);
+      mCodec = nullptr;
+      mSample = nullptr;
+    }
+
+  private:
+    void ReleaseOutput(bool aToRender)
+    {
+      if (mCodec && mSample) {
+        mCodec->ReleaseOutput(mSample, aToRender);
+      }
+    }
+
+    java::CodecProxy::GlobalRef mCodec;
+    java::Sample::GlobalRef mSample;
+  };
+
   class CallbacksSupport final : public JavaCallbacksSupport
   {
   public:
@@ -168,6 +202,9 @@ public:
                                                   mDecoder->mConfig.mDisplay.width,
                                                   mDecoder->mConfig.mDisplay.height));
 
+        UniquePtr<VideoData::Listener> listener(new RenderOrReleaseOutput(mDecoder->mJavaDecoder, aSample));
+        v->SetListener(Move(listener));
+
         mDecoderCallback->Output(v);
       }
 

mobile/android/base/java/org/mozilla/gecko/media/CodecProxy.java

@@ -18,6 +18,8 @@ import org.mozilla.gecko.mozglue.JNIObject;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
 
 // Proxy class of ICodec binder.
 public final class CodecProxy {
@@ -29,6 +31,7 @@ public final class CodecProxy {
     private Surface mOutputSurface;
     private CallbacksForwarder mCallbacks;
     private String mRemoteDrmStubId;
+    private Queue<Sample> mSurfaceOutputs = new ConcurrentLinkedQueue<>();
 
     public interface Callbacks {
         void onInputExhausted();
@@ -67,9 +70,18 @@ public final class CodecProxy {
 
         @Override
         public void onOutput(Sample sample) throws RemoteException {
-            mCallbacks.onOutput(sample);
-            mRemote.releaseOutput(sample, true);
-            sample.dispose();
+            if (mOutputSurface != null) {
+                // Don't render to surface just yet. Callback will make that happen when it's time.
+                if (!sample.isEOS() || sample.info.size > 0) {
+                    mSurfaceOutputs.offer(sample);
+                }
+                mCallbacks.onOutput(sample);
+            } else {
+                // Non-surface output needs no rendering.
+                mCallbacks.onOutput(sample);
+                mRemote.releaseOutput(sample, false);
+                sample.dispose();
+            }
         }
 
         @Override
@@ -196,6 +208,21 @@ public final class CodecProxy {
             return true;
         }
         if (DEBUG) Log.d(LOGTAG, "release " + this);
+
+        if (!mSurfaceOutputs.isEmpty()) {
+            // Flushing output buffers to surface may cause some frames to be skipped and
+            // should not happen unless caller release codec before processing all buffers.
+            Log.w(LOGTAG, "release codec when " + mSurfaceOutputs.size() + " output buffers unhandled");
+            try {
+                for (Sample s : mSurfaceOutputs) {
+                    mRemote.releaseOutput(s, true);
+                }
+            } catch (RemoteException e) {
+                e.printStackTrace();
+            }
+            mSurfaceOutputs.clear();
+        }
+
         try {
             RemoteManager.getInstance().releaseCodec(this);
         } catch (DeadObjectException e) {
@@ -207,7 +234,32 @@ public final class CodecProxy {
         return true;
     }
 
-    public synchronized void reportError(boolean fatal) {
+    @WrapForJNI
+    public synchronized boolean releaseOutput(Sample sample, boolean render) {
+        if (!mSurfaceOutputs.remove(sample)) {
+            if (mRemote != null) Log.w(LOGTAG, "already released: " + sample);
+            return true;
+        }
+
+        if (mRemote == null) {
+            Log.w(LOGTAG, "codec already ended");
+            sample.dispose();
+            return true;
+        }
+
+        if (DEBUG && !render) Log.d(LOGTAG, "drop output:" + sample.info.presentationTimeUs);
+
+        try {
+            mRemote.releaseOutput(sample, render);
+        } catch (RemoteException e) {
+            e.printStackTrace();
+        }
+        sample.dispose();
+
+        return true;
+    }
+
+    /* package */ synchronized void reportError(boolean fatal) {
         mCallbacks.reportError(fatal);
     }
 }

widget/android/fennec/FennecJNIWrappers.cpp

@@ -221,6 +221,14 @@ auto CodecProxy::Release() const -> bool
     return mozilla::jni::Method<Release_t>::Call(CodecProxy::mCtx, nullptr);
 }
 
+constexpr char CodecProxy::ReleaseOutput_t::name[];
+constexpr char CodecProxy::ReleaseOutput_t::signature[];
+
+auto CodecProxy::ReleaseOutput(mozilla::jni::Object::Param a0, bool a1) const -> bool
+{
+    return mozilla::jni::Method<ReleaseOutput_t>::Call(CodecProxy::mCtx, nullptr, a0, a1);
+}
+
 const char CodecProxy::NativeCallbacks::name[] =
         "org/mozilla/gecko/media/CodecProxy$NativeCallbacks";
 

widget/android/fennec/FennecJNIWrappers.h

@@ -765,6 +765,27 @@ public:
 
     auto Release() const -> bool;
 
+    struct ReleaseOutput_t {
+        typedef CodecProxy Owner;
+        typedef bool ReturnType;
+        typedef bool SetterType;
+        typedef mozilla::jni::Args<
+                mozilla::jni::Object::Param,
+                bool> Args;
+        static constexpr char name[] = "releaseOutput";
+        static constexpr char signature[] =
+                "(Lorg/mozilla/gecko/media/Sample;Z)Z";
+        static const bool isStatic = false;
+        static const mozilla::jni::ExceptionMode exceptionMode =
+                mozilla::jni::ExceptionMode::ABORT;
+        static const mozilla::jni::CallingThread callingThread =
+                mozilla::jni::CallingThread::ANY;
+        static const mozilla::jni::DispatchTarget dispatchTarget =
+                mozilla::jni::DispatchTarget::CURRENT;
+    };
+
+    auto ReleaseOutput(mozilla::jni::Object::Param, bool) const -> bool;
+
     static const mozilla::jni::CallingThread callingThread =
             mozilla::jni::CallingThread::ANY;