Bug 1263384: validate input frames against configured resolution in vp8 r=rillian

MozReview-Commit-ID: BxDCnJe0mzs

media/libvpx/input_frame_validation.patch

@@ -0,0 +1,46 @@
+# HG changeset patch
+# User Randell Jesup <rjesup@jesup.org>
+# Parent  1b77af186da211485fa9c5573d843d96c708a829
+Bug 1263384: validate input frames against configured resolution in vp8 r=rillian
+
+MozReview-Commit-ID: BxDCnJe0mzs
+
+diff --git a/media/libvpx/vp8/vp8_cx_iface.c b/media/libvpx/vp8/vp8_cx_iface.c
+--- a/media/libvpx/vp8/vp8_cx_iface.c
++++ b/media/libvpx/vp8/vp8_cx_iface.c
+@@ -916,21 +916,30 @@ static vpx_codec_err_t vp8e_encode(vpx_c
+         /* vp8 use 10,000,000 ticks/second as time stamp */
+         dst_time_stamp    = pts * 10000000 * ctx->cfg.g_timebase.num / ctx->cfg.g_timebase.den;
+         dst_end_time_stamp = (pts + duration) * 10000000 * ctx->cfg.g_timebase.num / ctx->cfg.g_timebase.den;
+ 
+         if (img != NULL)
+         {
+             res = image2yuvconfig(img, &sd);
+ 
+-            if (vp8_receive_raw_frame(ctx->cpi, ctx->next_frame_flag | lib_flags,
+-                                      &sd, dst_time_stamp, dst_end_time_stamp))
+-            {
+-                VP8_COMP *cpi = (VP8_COMP *)ctx->cpi;
+-                res = update_error_state(ctx, &cpi->common.error);
++            if (sd.y_width != ctx->cfg.g_w || sd.y_height != ctx->cfg.g_h) {
++                /* from vp8_encoder.h for g_w/g_h:
++                   "Note that the frames passed as input to the encoder must have this resolution"
++                */
++                ctx->base.err_detail = "Invalid input frame resolution";
++                res = VPX_CODEC_INVALID_PARAM;
++            } else {
++
++                if (vp8_receive_raw_frame(ctx->cpi, ctx->next_frame_flag | lib_flags,
++                                          &sd, dst_time_stamp, dst_end_time_stamp))
++                {
++                    VP8_COMP *cpi = (VP8_COMP *)ctx->cpi;
++                    res = update_error_state(ctx, &cpi->common.error);
++                }
+             }
+ 
+             /* reset for next frame */
+             ctx->next_frame_flag = 0;
+         }
+ 
+         cx_data = ctx->cx_data;
+         cx_data_sz = ctx->cx_data_sz;

media/libvpx/update.py

@@ -604,6 +604,8 @@ def apply_patches():
     os.system("patch -p3 < cast-char-to-uint-before-shift.patch")
     # Bug 1237848 - Check lookahead ctx
     os.system("patch -p3 < 1237848-check-lookahead-ctx.patch")
+    # Bug 1263384 - Check input frame resolution
+    os.system("patch -p3 < input_frame_validation.patch")
 
 def update_readme(commit):
     with open('README_MOZILLA') as f:

media/libvpx/vp8/vp8_cx_iface.c

@@ -921,11 +921,19 @@ static vpx_codec_err_t vp8e_encode(vpx_codec_alg_priv_t  *ctx,
         {
             res = image2yuvconfig(img, &sd);
 
-            if (vp8_receive_raw_frame(ctx->cpi, ctx->next_frame_flag | lib_flags,
-                                      &sd, dst_time_stamp, dst_end_time_stamp))
-            {
-                VP8_COMP *cpi = (VP8_COMP *)ctx->cpi;
-                res = update_error_state(ctx, &cpi->common.error);
+            if (sd.y_width != ctx->cfg.g_w || sd.y_height != ctx->cfg.g_h) {
+                /* from vp8_encoder.h for g_w/g_h:
+                   "Note that the frames passed as input to the encoder must have this resolution"
+                */
+                ctx->base.err_detail = "Invalid input frame resolution";
+                res = VPX_CODEC_INVALID_PARAM;
+            } else {
+                if (vp8_receive_raw_frame(ctx->cpi, ctx->next_frame_flag | lib_flags,
+                                          &sd, dst_time_stamp, dst_end_time_stamp))
+                {
+                    VP8_COMP *cpi = (VP8_COMP *)ctx->cpi;
+                    res = update_error_state(ctx, &cpi->common.error);
+                }
             }
 
             /* reset for next frame */