Fix some crashes in sb8x8 experiment.

Change-Id: I390bb1cedc835f439fd5dd6cda6572b29cbb139c

vp9/common/vp9_rtcd_defs.sh

@@ -342,6 +342,12 @@ specialize vp9_variance8x8 mmx sse2
 vp9_variance8x8_sse2=vp9_variance8x8_wmt
 vp9_variance8x8_mmx=vp9_variance8x8_mmx
 
+prototype unsigned int vp9_variance8x4 "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse"
+specialize vp9_variance8x4
+
+prototype unsigned int vp9_variance4x8 "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse"
+specialize vp9_variance4x8
+
 prototype unsigned int vp9_variance4x4 "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse"
 specialize vp9_variance4x4 mmx sse2
 vp9_variance4x4_sse2=vp9_variance4x4_wmt

vp9/encoder/vp9_onyx_if.c

@@ -1647,6 +1647,12 @@ VP9_PTR vp9_create_compressor(VP9_CONFIG *oxcf) {
   BFP(BLOCK_8X8, vp9_sad8x8, vp9_variance8x8, vp9_sub_pixel_variance8x8,
       NULL, NULL, NULL, vp9_sad8x8x3, vp9_sad8x8x8, vp9_sad8x8x4d)
 
+#if CONFIG_SB8X8
+  BFP(BLOCK_4X8, NULL, vp9_variance4x8, NULL,
+      NULL, NULL, NULL, NULL, NULL, NULL)
+  BFP(BLOCK_8X4, NULL, vp9_variance8x4, NULL,
+      NULL, NULL, NULL, NULL, NULL, NULL)
+#endif
   BFP(BLOCK_4X4, vp9_sad4x4, vp9_variance4x4, vp9_sub_pixel_variance4x4,
       NULL, NULL, NULL, vp9_sad4x4x3, vp9_sad4x4x8, vp9_sad4x4x4d)
 

vp9/encoder/vp9_onyx_int.h

@@ -283,6 +283,8 @@ typedef struct {
 enum BlockSize {
 #if CONFIG_SB8X8
   BLOCK_4X4,
+  BLOCK_4X8,
+  BLOCK_8X4,
   BLOCK_8X8,
   BLOCK_8X16,
   BLOCK_16X8,

vp9/encoder/vp9_rdopt.c

@@ -1874,7 +1874,7 @@ static void rd_check_segment_txsize(VP9_COMP *cpi, MACROBLOCK *x,
           raster_block_offset_uint8(&x->e_mbd, BLOCK_SIZE_SB8X8, 0, n,
                                     x->plane[0].src.buf,
                                     x->plane[0].src.stride);
-          assert(((intptr_t)x->e_mbd.plane[0].pre[0].buf & 0xf) == 0);
+          assert(((intptr_t)x->e_mbd.plane[0].pre[0].buf & 0x7) == 0);
           x->e_mbd.plane[0].pre[0].buf =
           raster_block_offset_uint8(&x->e_mbd, BLOCK_SIZE_SB8X8, 0, n,
                                     x->e_mbd.plane[0].pre[0].buf,
@@ -3295,6 +3295,11 @@ static enum BlockSize y_to_uv_block_size(enum BlockSize bs) {
     case BLOCK_32X16: return BLOCK_16X8;
     case BLOCK_16X32: return BLOCK_8X16;
     case BLOCK_16X16: return BLOCK_8X8;
+#if CONFIG_SB8X8
+    case BLOCK_16X8:  return BLOCK_8X4;
+    case BLOCK_8X16:  return BLOCK_4X8;
+    case BLOCK_8X8:   return BLOCK_4X4;
+#endif
     default:
       assert(0);
       return -1;
@@ -3310,6 +3315,11 @@ static enum BlockSize y_bsizet_to_block_size(BLOCK_SIZE_TYPE bs) {
     case BLOCK_SIZE_SB32X16: return BLOCK_32X16;
     case BLOCK_SIZE_SB16X32: return BLOCK_16X32;
     case BLOCK_SIZE_MB16X16: return BLOCK_16X16;
+#if CONFIG_SB8X8
+    case BLOCK_SIZE_SB16X8:  return BLOCK_16X8;
+    case BLOCK_SIZE_SB8X16:  return BLOCK_8X16;
+    case BLOCK_SIZE_SB8X8:   return BLOCK_8X8;
+#endif
     default:
       assert(0);
       return -1;
@@ -4927,7 +4937,9 @@ int64_t vp9_rd_pick_inter_mode_sb(VP9_COMP *cpi, MACROBLOCK *x,
   if (cpi->Speed == 0
       || (cpi->Speed > 0 && (ref_frame_mask & (1 << INTRA_FRAME)))) {
     mbmi->mode = DC_PRED;
-    for (i = 0; i <= ((bsize < BLOCK_SIZE_SB64X64) ? TX_16X16 : TX_32X32);
+    for (i = 0; i <= (bsize < BLOCK_SIZE_MB16X16 ? TX_4X4 :
+                      (bsize < BLOCK_SIZE_SB32X32 ? TX_8X8 :
+                       (bsize < BLOCK_SIZE_SB64X64 ? TX_16X16 : TX_32X32)));
          i++) {
       mbmi->txfm_size = i;
       rd_pick_intra_sbuv_mode(cpi, x, &rate_uv_intra[i], &rate_uv_tokenonly[i],
@@ -5097,6 +5109,8 @@ int64_t vp9_rd_pick_inter_mode_sb(VP9_COMP *cpi, MACROBLOCK *x,
                       bsize, txfm_cache);
 
       uv_tx = mbmi->txfm_size;
+      if (bsize < BLOCK_SIZE_MB16X16 && uv_tx == TX_8X8)
+        uv_tx = TX_4X4;
       if (bsize < BLOCK_SIZE_SB32X32 && uv_tx == TX_16X16)
         uv_tx = TX_8X8;
       else if (bsize < BLOCK_SIZE_SB64X64 && uv_tx == TX_32X32)

vp9/encoder/vp9_variance_c.c

@@ -239,6 +239,32 @@ unsigned int vp9_variance8x8_c(const uint8_t *src_ptr,
   return (var - (((unsigned int)avg * avg) >> 6));
 }
 
+unsigned int vp9_variance8x4_c(const uint8_t *src_ptr,
+                               int  source_stride,
+                               const uint8_t *ref_ptr,
+                               int  recon_stride,
+                               unsigned int *sse) {
+  unsigned int var;
+  int avg;
+
+  variance(src_ptr, source_stride, ref_ptr, recon_stride, 8, 4, &var, &avg);
+  *sse = var;
+  return (var - (((unsigned int)avg * avg) >> 5));
+}
+
+unsigned int vp9_variance4x8_c(const uint8_t *src_ptr,
+                               int  source_stride,
+                               const uint8_t *ref_ptr,
+                               int  recon_stride,
+                               unsigned int *sse) {
+  unsigned int var;
+  int avg;
+
+  variance(src_ptr, source_stride, ref_ptr, recon_stride, 4, 8, &var, &avg);
+  *sse = var;
+  return (var - (((unsigned int)avg * avg) >> 5));
+}
+
 unsigned int vp9_variance4x4_c(const uint8_t *src_ptr,
                                int  source_stride,
                                const uint8_t *ref_ptr,