Make switchable filter search subsampling-aware

Makes the temporary storage of the filtered data agnostic to
the number of planes and how they're subsampled.

Change-Id: I12f352cd69a47ebe1ac622af30db29b49becb7f4

vp9/encoder/vp9_rdopt.c

@@ -1852,9 +1852,7 @@ static int64_t handle_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
   int_mv cur_mv[2];
   int_mv ref_mv[2];
   int64_t this_rd = 0;
-  unsigned char tmp_ybuf[64 * 64];
+  unsigned char tmp_buf[MAX_MB_PLANE][64 * 64];
-  unsigned char tmp_ubuf[32 * 32];
-  unsigned char tmp_vbuf[32 * 32];
   int pred_exists = 0;
   int interpolating_intpel_seen = 0;
   int intpel_mv;
@@ -2032,19 +2030,17 @@ static int64_t handle_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
       if ((cm->mcomp_filter_type == SWITCHABLE && newbest) ||
           (cm->mcomp_filter_type != SWITCHABLE &&
            cm->mcomp_filter_type == mbmi->interp_filter)) {
-        int i;
+        int p;
-        for (i = 0; i < MI_SIZE * bh; ++i)
+
-          vpx_memcpy(tmp_ybuf + i * MI_SIZE * bw,
+        for (p = 0; p < MAX_MB_PLANE; p++) {
-                     xd->plane[0].dst.buf + i * xd->plane[0].dst.stride,
+          const int y = (MI_SIZE * bh) >> xd->plane[p].subsampling_y;
-                     sizeof(unsigned char) * MI_SIZE * bw);
+          const int x = (MI_SIZE * bw) >> xd->plane[p].subsampling_x;
-        for (i = 0; i < MI_UV_SIZE * bh; ++i)
+          int i;
-          vpx_memcpy(tmp_ubuf + i * MI_UV_SIZE * bw,
+
-                     xd->plane[1].dst.buf + i * xd->plane[1].dst.stride,
+          for (i = 0; i < y; i++)
-                     sizeof(unsigned char) * MI_UV_SIZE * bw);
+            vpx_memcpy(&tmp_buf[p][64 * i],
-        for (i = 0; i < MI_UV_SIZE * bh; ++i)
+                       xd->plane[p].dst.buf + i * xd->plane[p].dst.stride, x);
-          vpx_memcpy(tmp_vbuf + i * MI_UV_SIZE * bw,
+        }
-                     xd->plane[2].dst.buf + i * xd->plane[2].dst.stride,
-                     sizeof(unsigned char) * MI_UV_SIZE * bw);
         pred_exists = 1;
       }
       interpolating_intpel_seen |= is_intpel_interp;
@@ -2058,18 +2054,17 @@ static int64_t handle_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
 
 
   if (pred_exists) {
-    for (i = 0; i < bh * MI_SIZE; ++i)
+    int p;
-      vpx_memcpy(xd->plane[0].dst.buf + i * xd->plane[0].dst.stride,
+
-                 tmp_ybuf + i * bw * MI_SIZE,
+    for (p = 0; p < MAX_MB_PLANE; p++) {
-                 sizeof(unsigned char) * bw * MI_SIZE);
+      const int y = (MI_SIZE * bh) >> xd->plane[p].subsampling_y;
-    for (i = 0; i < bh * MI_UV_SIZE; ++i)
+      const int x = (MI_SIZE * bw) >> xd->plane[p].subsampling_x;
-      vpx_memcpy(xd->plane[1].dst.buf + i * xd->plane[1].dst.stride,
+      int i;
-                 tmp_ubuf + i * bw * MI_UV_SIZE,
+
-                 sizeof(unsigned char) * bw * MI_UV_SIZE);
+      for (i = 0; i < y; i++)
-    for (i = 0; i < bh * MI_UV_SIZE; ++i)
+        vpx_memcpy(xd->plane[p].dst.buf + i * xd->plane[p].dst.stride,
-      vpx_memcpy(xd->plane[2].dst.buf + i * xd->plane[2].dst.stride,
+                   &tmp_buf[p][64 * i], x);
-                 tmp_vbuf + i * bw * MI_UV_SIZE,
+    }
-                 sizeof(unsigned char) * bw * MI_UV_SIZE);
   } else {
     // Handles the special case when a filter that is not in the
     // switchable list (ex. bilinear, 6-tap) is indicated at the frame level