bug 1422239 - relax the resolution limitation of WMF H264 decoder; r=jya

https://msdn.microsoft.com/en-us/library/windows/desktop/dd797815(v=vs.85).aspx
Relax the resolution limitation from "width <= 4096 and height <= 2304" to "any width and height combination as long as the total pixel count is under 4096x2304".

MozReview-Commit-ID: 5wHiJfLaJkp

--HG--
extra : rebase_source : 11bf99d0eb3b50ea0199a7f65e0491e43318d29c

dom/media/platforms/wmf/WMFVideoMFTManager.cpp

@@ -558,7 +558,7 @@ MediaResult
 WMFVideoMFTManager::ValidateVideoInfo()
 {
   if (mStreamType != H264 ||
-      gfxPrefs::PDMWMFAllowUnsupportedResolutions()) {
+    gfxPrefs::PDMWMFAllowUnsupportedResolutions()) {
     return NS_OK;
   }
 
@@ -568,12 +568,12 @@ WMFVideoMFTManager::ValidateVideoInfo()
   // might have maximum resolution limitation.
   // https://msdn.microsoft.com/en-us/library/windows/desktop/dd797815(v=vs.85).aspx
   const bool Is4KCapable = IsWin8OrLater() || IsWin7H264Decoder4KCapable();
-  static const int32_t MIN_H264_FRAME_DIMENSION = 48;
-  static const int32_t MAX_H264_FRAME_WIDTH = Is4KCapable ? 4096 : 1920;
-  static const int32_t MAX_H264_FRAME_HEIGHT = Is4KCapable ? 2304 : 1088;
+  static const int32_t MAX_H264_PIXEL_COUNT =
+    Is4KCapable ? 4096 * 2304 : 1920 * 1088;
+  const CheckedInt32 pixelCount =
+    CheckedInt32(mVideoInfo.mImage.width) * mVideoInfo.mImage.height;
 
-  if (mVideoInfo.mImage.width > MAX_H264_FRAME_WIDTH  ||
-      mVideoInfo.mImage.height > MAX_H264_FRAME_HEIGHT) {
+  if (!pixelCount.isValid() || pixelCount.value() > MAX_H264_PIXEL_COUNT) {
     mIsValid = false;
     return MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
                        RESULT_DETAIL("Can't decode H.264 stream because its "
@@ -581,7 +581,6 @@ WMFVideoMFTManager::ValidateVideoInfo()
   }
 
   return NS_OK;
-
 }
 
 already_AddRefed<MFTDecoder>