Refactor mbmi->inter_tx_size to 2D array.

This is in preparation of increasing the superblock size.

Change-Id: I9197e397399fbe8aec1178a45ea0337dd90412d7

vp10/common/blockd.h

@@ -167,9 +167,9 @@ typedef struct {
   PREDICTION_MODE mode;
   TX_SIZE tx_size;
 #if CONFIG_VAR_TX
-  // TODO(jingning): This effectively assigned 64 entries for each 8x8 block.
+  // TODO(jingning): This effectively assigned an entry for each 8x8 block.
   // Apparently it takes much more space than needed.
-  TX_SIZE inter_tx_size[64];
+  TX_SIZE inter_tx_size[MI_BLOCK_SIZE][MI_BLOCK_SIZE];
 #endif
   int8_t skip;
   int8_t has_no_coeffs;

vp10/common/loopfilter.c

@@ -1276,9 +1276,9 @@ void vp10_filter_block_plane_non420(VP10_COMMON *cm,
 #if CONFIG_VAR_TX
       if (is_inter_block(mbmi) && !mbmi->skip)
         tx_size = (plane->plane_type == PLANE_TYPE_UV) ?
-            get_uv_tx_size_impl(mbmi->inter_tx_size[blk_row * 8 + blk_col],
+            get_uv_tx_size_impl(mbmi->inter_tx_size[blk_row][ blk_col],
                                 sb_type, ss_x, ss_y) :
-            mbmi->inter_tx_size[blk_row * 8 + blk_col];
+            mbmi->inter_tx_size[blk_row][blk_col];
 
       tx_size_r = VPXMIN(tx_size, cm->above_txfm_context[mi_col + c]);
       tx_size_c = VPXMIN(tx_size, cm->left_txfm_context[(mi_row + r) & 0x07]);

vp10/common/pred_common.h

@@ -192,9 +192,9 @@ static void update_tx_counts(VP10_COMMON *cm, MACROBLOCKD *xd,
                              TX_SIZE max_tx_size, int ctx) {
   const struct macroblockd_plane *const pd = &xd->plane[0];
   const BLOCK_SIZE bsize = txsize_to_bsize[tx_size];
-  int tx_idx = (blk_row >> (1 - pd->subsampling_y)) * 8 +
-               (blk_col >> (1 - pd->subsampling_x));
-  TX_SIZE plane_tx_size = mbmi->inter_tx_size[tx_idx];
+  const int tx_row = blk_row >> (1 - pd->subsampling_y);
+  const int tx_col = blk_col >> (1 - pd->subsampling_x);
+  const TX_SIZE plane_tx_size = mbmi->inter_tx_size[tx_row][tx_col];
   int max_blocks_high = num_4x4_blocks_high_lookup[plane_bsize];
   int max_blocks_wide = num_4x4_blocks_wide_lookup[plane_bsize];
 

vp10/decoder/decodeframe.c

@@ -307,11 +307,11 @@ static void decode_reconstruct_tx(MACROBLOCKD *const xd, vpx_reader *r,
                                   TX_SIZE tx_size, int *eob_total) {
   const struct macroblockd_plane *const pd = &xd->plane[plane];
   const BLOCK_SIZE bsize = txsize_to_bsize[tx_size];
-  int tx_idx = (blk_row >> (1 - pd->subsampling_y)) * 8 +
-               (blk_col >> (1 - pd->subsampling_x));
-  TX_SIZE plane_tx_size = plane ?
-      get_uv_tx_size_impl(mbmi->inter_tx_size[tx_idx], bsize, 0, 0) :
-      mbmi->inter_tx_size[tx_idx];
+  const int tx_row = blk_row >> (1 - pd->subsampling_y);
+  const int tx_col = blk_col >> (1 - pd->subsampling_x);
+  const TX_SIZE plane_tx_size = plane ?
+      get_uv_tx_size_impl(mbmi->inter_tx_size[tx_row][tx_col], bsize, 0, 0) :
+      mbmi->inter_tx_size[tx_row][tx_col];
   int max_blocks_high = num_4x4_blocks_high_lookup[plane_bsize];
   int max_blocks_wide = num_4x4_blocks_wide_lookup[plane_bsize];
 

vp10/decoder/decodemv.c

@@ -213,12 +213,15 @@ static void read_tx_size_inter(VP10_COMMON *cm, MACROBLOCKD *xd,
                                TX_SIZE tx_size, int blk_row, int blk_col,
                                vpx_reader *r) {
   int is_split = 0;
-  const int tx_idx = (blk_row >> 1) * 8 + (blk_col >> 1);
+  const int tx_row = blk_row >> 1;
+  const int tx_col = blk_col >> 1;
   int max_blocks_high = num_4x4_blocks_high_lookup[mbmi->sb_type];
   int max_blocks_wide = num_4x4_blocks_wide_lookup[mbmi->sb_type];
-  int ctx = txfm_partition_context(xd->above_txfm_context + (blk_col >> 1),
-                                   xd->left_txfm_context + (blk_row >> 1),
+  int ctx = txfm_partition_context(xd->above_txfm_context + tx_col,
+                                   xd->left_txfm_context + tx_row,
                                    tx_size);
+  TX_SIZE (*const inter_tx_size)[MI_BLOCK_SIZE] =
+    (TX_SIZE (*)[MI_BLOCK_SIZE])&mbmi->inter_tx_size[tx_row][tx_col];
 
   if (xd->mb_to_bottom_edge < 0)
     max_blocks_high += xd->mb_to_bottom_edge >> 5;
@@ -239,10 +242,10 @@ static void read_tx_size_inter(VP10_COMMON *cm, MACROBLOCKD *xd,
       ++counts->txfm_partition[ctx][1];
 
     if (tx_size == TX_8X8) {
-      mbmi->inter_tx_size[tx_idx] = TX_4X4;
-      mbmi->tx_size = mbmi->inter_tx_size[tx_idx];
-      txfm_partition_update(xd->above_txfm_context + (blk_col >> 1),
-                            xd->left_txfm_context + (blk_row >> 1), TX_4X4);
+      inter_tx_size[0][0] = TX_4X4;
+      mbmi->tx_size = TX_4X4;
+      txfm_partition_update(xd->above_txfm_context + tx_col,
+                            xd->left_txfm_context + tx_row, TX_4X4);
       return;
     }
 
@@ -256,15 +259,15 @@ static void read_tx_size_inter(VP10_COMMON *cm, MACROBLOCKD *xd,
     }
   } else {
     int idx, idy;
-    mbmi->inter_tx_size[tx_idx] = tx_size;
+    inter_tx_size[0][0] = tx_size;
     for (idy = 0; idy < (1 << tx_size) / 2; ++idy)
       for (idx = 0; idx < (1 << tx_size) / 2; ++idx)
-        mbmi->inter_tx_size[tx_idx + (idy << 3) + idx] = tx_size;
-    mbmi->tx_size = mbmi->inter_tx_size[tx_idx];
+        inter_tx_size[idy][idx] = tx_size;
+    mbmi->tx_size = tx_size;
     if (counts)
       ++counts->txfm_partition[ctx][0];
-    txfm_partition_update(xd->above_txfm_context + (blk_col >> 1),
-                          xd->left_txfm_context + (blk_row >> 1), tx_size);
+    txfm_partition_update(xd->above_txfm_context + tx_col,
+                          xd->left_txfm_context + tx_row, tx_size);
   }
 }
 #endif
@@ -1565,7 +1568,7 @@ static void read_inter_frame_mode_info(VP10Decoder *const pbi,
         int idx, idy;
         for (idy = 0; idy < height; ++idy)
           for (idx = 0; idx < width; ++idx)
-            mbmi->inter_tx_size[(idy >> 1) * 8 + (idx >> 1)] = mbmi->tx_size;
+            mbmi->inter_tx_size[idy >> 1][idx >> 1] = mbmi->tx_size;
       }
 
       set_txfm_ctx(xd->left_txfm_context, mbmi->tx_size, xd->n8_h);
@@ -1584,7 +1587,7 @@ static void read_inter_frame_mode_info(VP10Decoder *const pbi,
     xd->mi[0]->mbmi.tx_size = xd->supertx_size;
     for (idy = 0; idy < height; ++idy)
       for (idx = 0; idx < width; ++idx)
-        xd->mi[0]->mbmi.inter_tx_size[(idy >> 1) * 8 + (idx >> 1)] =
+        xd->mi[0]->mbmi.inter_tx_size[idy >> 1][idx >> 1] =
             xd->supertx_size;
   }
 #endif  // CONFIG_VAR_TX

vp10/encoder/bitstream.c

@@ -266,11 +266,12 @@ static void write_tx_size_inter(const VP10_COMMON *cm,
                                 const MB_MODE_INFO *mbmi,
                                 TX_SIZE tx_size, int blk_row, int blk_col,
                                 vpx_writer *w) {
-  const int tx_idx = (blk_row >> 1) * 8 + (blk_col >> 1);
+  const int tx_row = blk_row >> 1;
+  const int tx_col = blk_col >> 1;
   int max_blocks_high = num_4x4_blocks_high_lookup[mbmi->sb_type];
   int max_blocks_wide = num_4x4_blocks_wide_lookup[mbmi->sb_type];
-  int ctx = txfm_partition_context(xd->above_txfm_context + (blk_col >> 1),
-                                   xd->left_txfm_context + (blk_row >> 1),
+  int ctx = txfm_partition_context(xd->above_txfm_context + tx_col,
+                                   xd->left_txfm_context + tx_row,
                                    tx_size);
 
   if (xd->mb_to_bottom_edge < 0)
@@ -281,10 +282,10 @@ static void write_tx_size_inter(const VP10_COMMON *cm,
   if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide)
      return;
 
-  if (tx_size == mbmi->inter_tx_size[tx_idx]) {
+  if (tx_size == mbmi->inter_tx_size[tx_row][tx_col]) {
     vpx_write(w, 0, cm->fc->txfm_partition_prob[ctx]);
-    txfm_partition_update(xd->above_txfm_context + (blk_col >> 1),
-                          xd->left_txfm_context + (blk_row >> 1), tx_size);
+    txfm_partition_update(xd->above_txfm_context + tx_col,
+                          xd->left_txfm_context + tx_row, tx_size);
   } else {
     const BLOCK_SIZE bsize = txsize_to_bsize[tx_size];
     int bsl = b_width_log2_lookup[bsize];
@@ -292,8 +293,8 @@ static void write_tx_size_inter(const VP10_COMMON *cm,
     vpx_write(w, 1, cm->fc->txfm_partition_prob[ctx]);
 
     if (tx_size == TX_8X8) {
-      txfm_partition_update(xd->above_txfm_context + (blk_col >> 1),
-                            xd->left_txfm_context + (blk_row >> 1), TX_4X4);
+      txfm_partition_update(xd->above_txfm_context + tx_col,
+                            xd->left_txfm_context + tx_row, TX_4X4);
       return;
     }
 
@@ -706,11 +707,11 @@ static void pack_txb_tokens(vpx_writer *w,
                            int blk_row, int blk_col, TX_SIZE tx_size) {
   const struct macroblockd_plane *const pd = &xd->plane[plane];
   const BLOCK_SIZE bsize = txsize_to_bsize[tx_size];
-  int tx_idx = (blk_row >> (1 - pd->subsampling_y)) * 8 +
-               (blk_col >> (1 - pd->subsampling_x));
-  TX_SIZE plane_tx_size = plane ?
-      get_uv_tx_size_impl(mbmi->inter_tx_size[tx_idx], bsize, 0, 0) :
-      mbmi->inter_tx_size[tx_idx];
+  const int tx_row = blk_row >> (1 - pd->subsampling_y);
+  const int tx_col = blk_col >> (1 - pd->subsampling_x);
+  const TX_SIZE plane_tx_size = plane ?
+      get_uv_tx_size_impl(mbmi->inter_tx_size[tx_row][tx_col], bsize, 0, 0) :
+      mbmi->inter_tx_size[tx_row][tx_col];
   int max_blocks_high = num_4x4_blocks_high_lookup[plane_bsize];
   int max_blocks_wide = num_4x4_blocks_wide_lookup[plane_bsize];
 

vp10/encoder/encodeframe.c

@@ -1287,7 +1287,7 @@ static void update_state_supertx(VP10_COMP *cpi, ThreadData *td,
     int idy, idx;
     for (idy = 0; idy < (1 << mtx) / 2; ++idy)
       for (idx = 0; idx < (1 << mtx) / 2; ++idx)
-        mbmi->inter_tx_size[(idy << 3) + idx] = mbmi->tx_size;
+        mbmi->inter_tx_size[idy][idx] = mbmi->tx_size;
   }
 #endif  // CONFIG_VAR_TX
 #if CONFIG_OBMC
@@ -4218,13 +4218,14 @@ static void update_txfm_count(MACROBLOCKD *xd,
                               FRAME_COUNTS *counts,
                               TX_SIZE tx_size, int blk_row, int blk_col) {
   MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
-  int tx_idx = (blk_row >> 1) * 8 + (blk_col >> 1);
+  const int tx_row = blk_row >> 1;
+  const int tx_col = blk_col >> 1;
   int max_blocks_high = num_4x4_blocks_high_lookup[mbmi->sb_type];
   int max_blocks_wide = num_4x4_blocks_wide_lookup[mbmi->sb_type];
-  int ctx = txfm_partition_context(xd->above_txfm_context + (blk_col >> 1),
-                                   xd->left_txfm_context + (blk_row >> 1),
+  int ctx = txfm_partition_context(xd->above_txfm_context + tx_col,
+                                   xd->left_txfm_context + tx_row,
                                    tx_size);
-  TX_SIZE plane_tx_size = mbmi->inter_tx_size[tx_idx];
+  const TX_SIZE plane_tx_size = mbmi->inter_tx_size[tx_row][tx_col];
 
   if (xd->mb_to_bottom_edge < 0)
     max_blocks_high += xd->mb_to_bottom_edge >> 5;
@@ -4237,8 +4238,8 @@ static void update_txfm_count(MACROBLOCKD *xd,
   if (tx_size == plane_tx_size) {
     ++counts->txfm_partition[ctx][0];
     mbmi->tx_size = tx_size;
-    txfm_partition_update(xd->above_txfm_context + (blk_col >> 1),
-                          xd->left_txfm_context + (blk_row >> 1), tx_size);
+    txfm_partition_update(xd->above_txfm_context + tx_col,
+                          xd->left_txfm_context + tx_row, tx_size);
   } else {
     BLOCK_SIZE bsize = txsize_to_bsize[tx_size];
     int bh = num_4x4_blocks_high_lookup[bsize];
@@ -4246,10 +4247,10 @@ static void update_txfm_count(MACROBLOCKD *xd,
     ++counts->txfm_partition[ctx][1];
 
     if (tx_size == TX_8X8) {
-      mbmi->inter_tx_size[tx_idx] = TX_4X4;
+      mbmi->inter_tx_size[tx_row][tx_col] = TX_4X4;
       mbmi->tx_size = TX_4X4;
-      txfm_partition_update(xd->above_txfm_context + (blk_col >> 1),
-                            xd->left_txfm_context + (blk_row >> 1), TX_4X4);
+      txfm_partition_update(xd->above_txfm_context + tx_col,
+                            xd->left_txfm_context + tx_row, TX_4X4);
       return;
     }
 
@@ -4285,10 +4286,11 @@ static void tx_partition_count_update(VP10_COMMON *cm,
 static void set_txfm_context(MACROBLOCKD *xd, TX_SIZE tx_size,
                              int blk_row, int blk_col) {
   MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
-  int tx_idx = (blk_row >> 1) * 8 + (blk_col >> 1);
+  const int tx_row = blk_row >> 1;
+  const int tx_col = blk_col >> 1;
   int max_blocks_high = num_4x4_blocks_high_lookup[mbmi->sb_type];
   int max_blocks_wide = num_4x4_blocks_wide_lookup[mbmi->sb_type];
-  TX_SIZE plane_tx_size = mbmi->inter_tx_size[tx_idx];
+  const TX_SIZE plane_tx_size = mbmi->inter_tx_size[tx_row][tx_col];
 
   if (xd->mb_to_bottom_edge < 0)
     max_blocks_high += xd->mb_to_bottom_edge >> 5;
@@ -4300,8 +4302,8 @@ static void set_txfm_context(MACROBLOCKD *xd, TX_SIZE tx_size,
 
   if (tx_size == plane_tx_size) {
     mbmi->tx_size = tx_size;
-    txfm_partition_update(xd->above_txfm_context + (blk_col >> 1),
-                          xd->left_txfm_context + (blk_row >> 1), tx_size);
+    txfm_partition_update(xd->above_txfm_context + tx_col,
+                          xd->left_txfm_context + tx_row, tx_size);
 
   } else {
     BLOCK_SIZE bsize = txsize_to_bsize[tx_size];
@@ -4309,10 +4311,10 @@ static void set_txfm_context(MACROBLOCKD *xd, TX_SIZE tx_size,
     int i;
 
     if (tx_size == TX_8X8) {
-      mbmi->inter_tx_size[tx_idx] = TX_4X4;
+      mbmi->inter_tx_size[tx_row][tx_col] = TX_4X4;
       mbmi->tx_size = TX_4X4;
-      txfm_partition_update(xd->above_txfm_context + (blk_col >> 1),
-                            xd->left_txfm_context + (blk_row >> 1), TX_4X4);
+      txfm_partition_update(xd->above_txfm_context + tx_col,
+                            xd->left_txfm_context + tx_row, TX_4X4);
       return;
     }
 

vp10/encoder/encodemb.c

@@ -558,12 +558,11 @@ static void encode_block_inter(int plane, int block, int blk_row, int blk_col,
   MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
   const BLOCK_SIZE bsize = txsize_to_bsize[tx_size];
   const struct macroblockd_plane *const pd = &xd->plane[plane];
-  int blk_idx = (blk_row >> (1 - pd->subsampling_y)) * 8 +
-                (blk_col >> (1 - pd->subsampling_x));
-  TX_SIZE plane_tx_size = plane ?
-      get_uv_tx_size_impl(mbmi->inter_tx_size[blk_idx], bsize,
-                          0, 0) :
-      mbmi->inter_tx_size[blk_idx];
+  const int tx_row = blk_row >> (1 - pd->subsampling_y);
+  const int tx_col = blk_col >> (1 - pd->subsampling_x);
+  const TX_SIZE plane_tx_size = plane ?
+      get_uv_tx_size_impl(mbmi->inter_tx_size[tx_row][tx_col], bsize, 0, 0) :
+      mbmi->inter_tx_size[tx_row][tx_col];
 
   int max_blocks_high = num_4x4_blocks_high_lookup[plane_bsize];
   int max_blocks_wide = num_4x4_blocks_wide_lookup[plane_bsize];

vp10/encoder/rdopt.c

@@ -2921,8 +2921,10 @@ static void select_tx_block(const VP10_COMP *cpi, MACROBLOCK *x,
   MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
   struct macroblock_plane *const p = &x->plane[plane];
   struct macroblockd_plane *const pd = &xd->plane[plane];
-  int tx_idx = (blk_row >> (1 - pd->subsampling_y)) * 8 +
-               (blk_col >> (1 - pd->subsampling_x));
+  const int tx_row = blk_row >> (1 - pd->subsampling_y);
+  const int tx_col = blk_col >> (1 - pd->subsampling_x);
+  TX_SIZE (*const inter_tx_size)[MI_BLOCK_SIZE] =
+    (TX_SIZE (*)[MI_BLOCK_SIZE])&mbmi->inter_tx_size[tx_row][tx_col];
   int max_blocks_high = num_4x4_blocks_high_lookup[plane_bsize];
   int max_blocks_wide = num_4x4_blocks_wide_lookup[plane_bsize];
   int64_t this_rd = INT64_MAX;
@@ -2985,7 +2987,7 @@ static void select_tx_block(const VP10_COMP *cpi, MACROBLOCK *x,
       x->token_costs[tx_size][pd->plane_type][1][0][0][coeff_ctx][EOB_TOKEN];
 
   if (cpi->common.tx_mode == TX_MODE_SELECT || tx_size == TX_4X4) {
-    mbmi->inter_tx_size[tx_idx] = tx_size;
+    inter_tx_size[0][0] = tx_size;
     vp10_tx_block_rd_b(cpi, x, tx_size, blk_row, blk_col, plane, block,
                        plane_bsize, coeff_ctx, rate, dist, bsse, skip);
 
@@ -3048,11 +3050,10 @@ static void select_tx_block(const VP10_COMP *cpi, MACROBLOCK *x,
       pta[i] = ptl[i] = !(tmp_eob == 0);
     txfm_partition_update(tx_above + (blk_col >> 1),
                           tx_left + (blk_row >> 1), tx_size);
-    mbmi->inter_tx_size[tx_idx] = tx_size;
-
+    inter_tx_size[0][0] = tx_size;
     for (idy = 0; idy < (1 << tx_size) / 2; ++idy)
       for (idx = 0; idx < (1 << tx_size) / 2; ++idx)
-        mbmi->inter_tx_size[tx_idx + (idy << 3) + idx] = tx_size;
+        inter_tx_size[idy][idx] = tx_size;
     mbmi->tx_size = tx_size;
     if (this_rd == INT64_MAX)
       *is_cost_valid = 0;
@@ -3152,7 +3153,7 @@ static void select_tx_type_yrd(const VP10_COMP *cpi, MACROBLOCK *x,
   vpx_prob skip_prob = vp10_get_skip_prob(cm, xd);
   int s0 = vp10_cost_bit(skip_prob, 0);
   int s1 = vp10_cost_bit(skip_prob, 1);
-  TX_SIZE best_tx_size[64];
+  TX_SIZE best_tx_size[MI_BLOCK_SIZE][MI_BLOCK_SIZE];
   TX_SIZE best_tx = TX_SIZES;
   uint8_t best_blk_skip[256];
   const int n4 = 1 << (num_pels_log2_lookup[bsize] - 4);
@@ -3267,14 +3268,14 @@ static void select_tx_type_yrd(const VP10_COMP *cpi, MACROBLOCK *x,
       memcpy(best_blk_skip, x->blk_skip[0], sizeof(best_blk_skip[0]) * n4);
       for (idy = 0; idy < xd->n8_h; ++idy)
         for (idx = 0; idx < xd->n8_w; ++idx)
-          best_tx_size[idy * 8 + idx] = mbmi->inter_tx_size[idy * 8 + idx];
+          best_tx_size[idy][idx] = mbmi->inter_tx_size[idy][idx];
     }
   }
 
   mbmi->tx_type = best_tx_type;
   for (idy = 0; idy < xd->n8_h; ++idy)
     for (idx = 0; idx < xd->n8_w; ++idx)
-      mbmi->inter_tx_size[idy * 8 + idx] = best_tx_size[idy * 8 + idx];
+      mbmi->inter_tx_size[idy][idx] = best_tx_size[idy][idx];
   mbmi->tx_size = best_tx;
   memcpy(x->blk_skip[0], best_blk_skip, sizeof(best_blk_skip[0]) * n4);
 }
@@ -3289,12 +3290,11 @@ static void tx_block_rd(const VP10_COMP *cpi, MACROBLOCK *x,
   struct macroblock_plane *const p = &x->plane[plane];
   struct macroblockd_plane *const pd = &xd->plane[plane];
   BLOCK_SIZE bsize = txsize_to_bsize[tx_size];
-  int tx_idx = (blk_row >> (1 - pd->subsampling_y)) * 8 +
-               (blk_col >> (1 - pd->subsampling_x));
-  TX_SIZE plane_tx_size = plane ?
-      get_uv_tx_size_impl(mbmi->inter_tx_size[tx_idx], bsize,
-                          0, 0) :
-      mbmi->inter_tx_size[tx_idx];
+  const int tx_row = blk_row >> (1 - pd->subsampling_y);
+  const int tx_col = blk_col >> (1 - pd->subsampling_x);
+  const TX_SIZE plane_tx_size = plane ?
+      get_uv_tx_size_impl(mbmi->inter_tx_size[tx_row][tx_col], bsize, 0, 0) :
+      mbmi->inter_tx_size[tx_row][tx_col];
 
   int max_blocks_high = num_4x4_blocks_high_lookup[plane_bsize];
   int max_blocks_wide = num_4x4_blocks_wide_lookup[plane_bsize];
@@ -6923,7 +6923,7 @@ static int64_t handle_inter_mode(VP10_COMP *cpi, MACROBLOCK *x,
                       bsize, ref_best_rd);
       for (idy = 0; idy < xd->n8_h; ++idy)
         for (idx = 0; idx < xd->n8_w; ++idx)
-          mbmi->inter_tx_size[idy * 8 + idx] = mbmi->tx_size;
+          mbmi->inter_tx_size[idy][idx] = mbmi->tx_size;
     }
 #else
     super_block_yrd(cpi, x, rate_y, &distortion_y, &skippable_y, psse,
@@ -9087,7 +9087,7 @@ void vp10_rd_pick_inter_mode_sub8x8(struct VP10_COMP *cpi,
     }
 
 #if CONFIG_VAR_TX
-    mbmi->inter_tx_size[0] = mbmi->tx_size;
+    mbmi->inter_tx_size[0][0] = mbmi->tx_size;
 #endif
 
     if (ref_frame == INTRA_FRAME) {

vp10/encoder/tokenize.c

@@ -565,11 +565,11 @@ void tokenize_tx(ThreadData *td, TOKENEXTRA **t,
   MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
   const struct macroblockd_plane *const pd = &xd->plane[plane];
   const BLOCK_SIZE bsize = txsize_to_bsize[tx_size];
-  int blk_idx = (blk_row >> (1 - pd->subsampling_y)) * 8 +
-                (blk_col >> (1 - pd->subsampling_x));
-  TX_SIZE plane_tx_size = plane ?
-      get_uv_tx_size_impl(mbmi->inter_tx_size[blk_idx], bsize, 0, 0) :
-      mbmi->inter_tx_size[blk_idx];
+  const int tx_row = blk_row >> (1 - pd->subsampling_y);
+  const int tx_col = blk_col >> (1 - pd->subsampling_x);
+  const TX_SIZE plane_tx_size = plane ?
+      get_uv_tx_size_impl(mbmi->inter_tx_size[tx_row][tx_col], bsize, 0, 0) :
+      mbmi->inter_tx_size[tx_row][tx_col];
 
   int max_blocks_high = num_4x4_blocks_high_lookup[plane_bsize];
   int max_blocks_wide = num_4x4_blocks_wide_lookup[plane_bsize];