Bug 1640416 - p2: sanity-check video encoder config size arguements. r=alwu

Differential Revision: https://phabricator.services.mozilla.com/D94510

dom/media/platforms/android/AndroidDataEncoder.cpp

@@ -37,6 +37,11 @@ extern LazyLogModule sPEMLog;
   } while (0)
 
 RefPtr<MediaDataEncoder::InitPromise> AndroidDataEncoder::Init() {
+  // Sanity-check the input size for Android software encoder fails to do it.
+  if (mConfig.mSize.width == 0 || mConfig.mSize.height == 0) {
+    return InitPromise::CreateAndReject(NS_ERROR_ILLEGAL_VALUE, __func__);
+  }
+
   return InvokeAsync(mTaskQueue, this, __func__,
                      &AndroidDataEncoder::ProcessInit);
 }

dom/media/platforms/android/AndroidDataEncoder.h

@@ -21,7 +21,10 @@ class AndroidDataEncoder final : public MediaDataEncoder {
   using Config = H264Config;
 
   AndroidDataEncoder(const Config& aConfig, RefPtr<TaskQueue> aTaskQueue)
-      : mConfig(aConfig), mTaskQueue(aTaskQueue) {}
+      : mConfig(aConfig), mTaskQueue(aTaskQueue) {
+    MOZ_ASSERT(mConfig.mSize.width > 0 && mConfig.mSize.height > 0);
+    MOZ_ASSERT(mTaskQueue);
+  }
   RefPtr<InitPromise> Init() override;
   RefPtr<EncodePromise> Encode(const MediaData* aSample) override;
   RefPtr<EncodePromise> Drain() override;

dom/media/platforms/apple/AppleVTEncoder.cpp

@@ -92,6 +92,10 @@ static bool SetProfileLevel(VTCompressionSessionRef& aSession,
 RefPtr<MediaDataEncoder::InitPromise> AppleVTEncoder::Init() {
   MOZ_ASSERT(!mInited, "Cannot initialize encoder again without shutting down");
 
+  if (mConfig.mSize.width == 0 || mConfig.mSize.height == 0) {
+    return InitPromise::CreateAndReject(NS_ERROR_ILLEGAL_VALUE, __func__);
+  }
+
   AutoCFRelease<CFDictionaryRef> spec(BuildEncoderSpec());
   AutoCFRelease<CFDictionaryRef> srcBufferAttr(
       BuildSourceImageBufferAttributes());

dom/media/platforms/apple/AppleVTEncoder.h

@@ -29,6 +29,7 @@ class AppleVTEncoder final : public MediaDataEncoder {
         mFramesCompleted(false),
         mError(NS_OK),
         mSession(nullptr) {
+    MOZ_ASSERT(mConfig.mSize.width > 0 && mConfig.mSize.height > 0);
     MOZ_ASSERT(mTaskQueue);
   }