Bug 1089423 - Catch MediaCodec exceptions r=gcp

dom/media/fmp4/android/AndroidDecoderModule.cpp

@@ -276,7 +276,11 @@ nsresult MediaCodecDataDecoder::InitDecoder(jobject aSurface)
     return NS_ERROR_FAILURE;
   }
 
-  mDecoder->Configure(mFormat->wrappedObject(), aSurface, nullptr, 0);
+  if (!mDecoder->Configure(mFormat->wrappedObject(), aSurface, nullptr, 0)) {
+    mCallback->Error();
+    return NS_ERROR_FAILURE;
+  }
+
   mDecoder->Start();
 
   ResetInputBuffers();

widget/android/GeneratedSDKWrappers.cpp

@@ -85,7 +85,7 @@ MediaCodec* MediaCodec::Wrap(jobject obj) {
     return ret;
 }
 
-void MediaCodec::Configure(jobject a0, jobject a1, jobject a2, int32_t a3) {
+bool MediaCodec::Configure(jobject a0, jobject a1, jobject a2, int32_t a3) {
     JNIEnv *env = GetJNIForThread();
     if (env->PushLocalFrame(3) != 0) {
         AndroidBridge::HandleUncaughtException(env);
@@ -99,8 +99,13 @@ void MediaCodec::Configure(jobject a0, jobject a1, jobject a2, int32_t a3) {
     args[3].i = a3;
 
     env->CallVoidMethodA(wrapped_obj, jConfigure, args);
-    AndroidBridge::HandleUncaughtException(env);
+    if (env->ExceptionCheck()) {
+        env->ExceptionClear();
+        env->PopLocalFrame(nullptr);
+        return false;
+    }
     env->PopLocalFrame(nullptr);
+    return true;
 }
 
 jobject MediaCodec::CreateByCodecName(const nsAString& a0) {
@@ -156,7 +161,11 @@ int32_t MediaCodec::DequeueInputBuffer(int64_t a0) {
     }
 
     int32_t temp = env->CallIntMethod(wrapped_obj, jDequeueInputBuffer, a0);
-    AndroidBridge::HandleUncaughtException(env);
+    if (env->ExceptionCheck()) {
+        env->ExceptionClear();
+        env->PopLocalFrame(nullptr);
+        return MEDIACODEC_EXCEPTION_INDEX;
+    }
     env->PopLocalFrame(nullptr);
     return temp;
 }
@@ -169,7 +178,11 @@ int32_t MediaCodec::DequeueOutputBuffer(jobject a0, int64_t a1) {
     }
 
     int32_t temp = env->CallIntMethod(wrapped_obj, jDequeueOutputBuffer, a0, a1);
-    AndroidBridge::HandleUncaughtException(env);
+    if (env->ExceptionCheck()) {
+        env->ExceptionClear();
+        env->PopLocalFrame(nullptr);
+        return MEDIACODEC_EXCEPTION_INDEX;
+    }
     env->PopLocalFrame(nullptr);
     return temp;
 }
@@ -311,7 +324,7 @@ void MediaCodec::SetVideoScalingMode(int32_t a0) {
     env->PopLocalFrame(nullptr);
 }
 
-void MediaCodec::Start() {
+bool MediaCodec::Start() {
     JNIEnv *env = GetJNIForThread();
     if (env->PushLocalFrame(0) != 0) {
         AndroidBridge::HandleUncaughtException(env);
@@ -319,8 +332,13 @@ void MediaCodec::Start() {
     }
 
     env->CallVoidMethod(wrapped_obj, jStart);
-    AndroidBridge::HandleUncaughtException(env);
+    if (env->ExceptionCheck()) {
+        env->ExceptionClear();
+        env->PopLocalFrame(nullptr);
+        return false;
+    }
     env->PopLocalFrame(nullptr);
+    return true;
 }
 
 void MediaCodec::Stop() {

widget/android/GeneratedSDKWrappers.h

@@ -17,6 +17,9 @@
 namespace mozilla {
 namespace widget {
 namespace android {
+
+#define MEDIACODEC_EXCEPTION_INDEX -255
+
 void InitSDKStubs(JNIEnv *jEnv);
 
 class MediaCodec : public AutoGlobalWrappedJavaObject {
@@ -24,7 +27,7 @@ public:
     static void InitStubs(JNIEnv *jEnv);
     static MediaCodec* Wrap(jobject obj);
     MediaCodec(jobject obj, JNIEnv* env) : AutoGlobalWrappedJavaObject(obj, env) {};
-    void Configure(jobject a0, jobject a1, jobject a2, int32_t a3);
+    bool Configure(jobject a0, jobject a1, jobject a2, int32_t a3);
     static jobject CreateByCodecName(const nsAString& a0);
     static jobject CreateDecoderByType(const nsAString& a0);
     static jobject CreateEncoderByType(const nsAString& a0);
@@ -40,7 +43,7 @@ public:
     void Release();
     void ReleaseOutputBuffer(int32_t a0, bool a1);
     void SetVideoScalingMode(int32_t a0);
-    void Start();
+    bool Start();
     void Stop();
     static int32_t getBUFFER_FLAG_CODEC_CONFIG();
     static int32_t getBUFFER_FLAG_END_OF_STREAM();