Bug 1274498 - Refuse to play H.264 files which have one or more dimensions less than 48 pixels. r=jya

We've had large numbers of shutdown hangs with the Windows H.264 decoder stuck
calling IMFTransform::ProcessOutput(), blocking shutdown. I can reproduce this
with videos with dimensions less than 32 pixels.

Chrome also encountered this with the WMF decoder:
https://bugs.chromium.org/p/chromium/issues/detail?id=373288

The WMF H.264 Decoder is documented to have a minimum resolution of 48x48 pixels.

So this patch causes us to reject H.264 files with either width or height less
than 48 pixels.

I have been able to play files down to 34x34 pixels on Windows 10, but it seems
safest to just follow the what's documented in MSDN, and reject files that are
smaller than the documented minimum.

MozReview-Commit-ID: 5peP6UGnAaB

--HG--
extra : rebase_source : 6e29812642bc3f8ca0f5b39b36064a6d50e09ea7

dom/media/platforms/wmf/WMFVideoMFTManager.cpp

@@ -91,6 +91,7 @@ WMFVideoMFTManager::WMFVideoMFTManager(
   , mNullOutputCount(0)
   , mGotValidOutputAfterNullOutput(false)
   , mGotExcessiveNullOutput(false)
+  , mIsValid(true)
   // mVideoStride, mVideoWidth, mVideoHeight, mUseHwAccel are initialized in
   // Init().
 {
@@ -370,9 +371,33 @@ WMFVideoMFTManager::InitializeDXVA(bool aForceD3D9)
   return mDXVA2Manager != nullptr;
 }
 
+bool
+WMFVideoMFTManager::ValidateVideoInfo()
+{
+  // The WMF H.264 decoder is documented to have a minimum resolution
+  // 48x48 pixels. We've observed the decoder working for output smaller than
+  // that, but on some output it hangs in IMFTransform::ProcessOutput(), so
+  // we just reject streams which are less than the documented minimum.
+  // https://msdn.microsoft.com/en-us/library/windows/desktop/dd797815(v=vs.85).aspx
+  static const int32_t MIN_H264_FRAME_DIMENSION = 48;
+  if (mStreamType == H264 &&
+      (mVideoInfo.mImage.width < MIN_H264_FRAME_DIMENSION ||
+       mVideoInfo.mImage.height < MIN_H264_FRAME_DIMENSION)) {
+    LogToBrowserConsole(NS_LITERAL_STRING(
+      "Can't decode H.264 stream with width or height less than 48 pixels."));
+    mIsValid = false;
+  }
+
+  return mIsValid;
+}
+
 bool
 WMFVideoMFTManager::Init()
 {
+  if (!ValidateVideoInfo()) {
+    return false;
+  }
+
   bool success = InitInternal(/* aForceD3D9 = */ false);
 
   if (success && mDXVA2Manager) {
@@ -484,6 +509,10 @@ WMFVideoMFTManager::SetDecoderMediaTypes()
 HRESULT
 WMFVideoMFTManager::Input(MediaRawData* aSample)
 {
+  if (!mIsValid) {
+    return E_FAIL;
+  }
+
   if (!mDecoder) {
     // This can happen during shutdown.
     return E_FAIL;
@@ -906,6 +935,7 @@ WMFVideoMFTManager::ConfigurationChanged(const TrackInfo& aConfig)
   MOZ_ASSERT(aConfig.GetAsVideoInfo());
   mVideoInfo = *aConfig.GetAsVideoInfo();
   mImageSize = mVideoInfo.mImage;
+  ValidateVideoInfo();
 }
 
 } // namespace mozilla

dom/media/platforms/wmf/WMFVideoMFTManager.h

@@ -51,6 +51,8 @@ public:
 
 private:
 
+  bool ValidateVideoInfo();
+
   bool InitializeDXVA(bool aForceD3D9);
 
   bool InitInternal(bool aForceD3D9);
@@ -101,6 +103,7 @@ private:
   uint32_t mNullOutputCount;
   bool mGotValidOutputAfterNullOutput;
   bool mGotExcessiveNullOutput;
+  bool mIsValid;
 };
 
 } // namespace mozilla

dom/media/test/manifest.js

@@ -533,6 +533,20 @@ var gErrorTests = [
   { name:"bogus.duh", type:"bogus/duh" }
 ];
 
+function IsWindowsVistaOrLater() {
+  var re = /Windows NT (\d+.\d)/;
+  var winver = manifestNavigator().userAgent.match(re);
+  return winver && winver.length == 2 && parseFloat(winver[1]) >= 6.0;
+}
+
+// Windows' H.264 decoder cannot handle H.264 streams with resolution
+// less than 48x48 pixels. We refuse to play and error on such streams.
+if (IsWindowsVistaOrLater() &&
+    manifestVideo().canPlayType('video/mp4; codecs="avc1.42E01E"')) {
+  gErrorTests = gErrorTests.concat({name: "red-46x48.mp4", type:"video/mp4"},
+                                   {name: "red-48x46.mp4", type:"video/mp4"});
+}
+
 // These are files that have nontrivial duration and are useful for seeking within.
 var gSeekTests = [
   { name:"r11025_s16_c1.wav", type:"audio/x-wav", duration:1.0 },
@@ -562,12 +576,6 @@ var gFastSeekTests = [
   { name:"bug516323.indexed.ogv", type:"video/ogg", keyframes:[0, 0.46, 3.06] },
 ];
 
-function IsWindows8OrLater() {
-  var re = /Windows NT (\d.\d)/;
-  var winver = manifestNavigator().userAgent.match(re);
-  return winver && winver.length == 2 && parseFloat(winver[1]) >= 6.2;
-}
-
 // These files are WebMs without cues. They're seekable within their buffered
 // ranges. If work renders WebMs fully seekable these files should be moved
 // into gSeekTests

dom/media/test/mochitest.ini

@@ -491,6 +491,10 @@ support-files =
   r16000_u8_c1_list.wav
   r16000_u8_c1_list.wav^headers^
   reactivate_helper.html
+  red-46x48.mp4
+  red-46x48.mp4^headers^
+  red-48x46.mp4
+  red-48x46.mp4^headers^
   redirect.sjs
   referer.sjs
   region.vtt

dom/media/test/red-46x48.mp4^headers^

@@ -0,0 +1 @@
+Cache-Control: no-store

dom/media/test/red-48x46.mp4^headers^

@@ -0,0 +1 @@
+Cache-Control: no-store