diff --git a/av1/encoder/block.h b/av1/encoder/block.h
index af7df7c1f..761075041 100644
--- a/av1/encoder/block.h
+++ b/av1/encoder/block.h
@@ -67,7 +67,6 @@ struct macroblock {
   MACROBLOCKD e_mbd;
   MB_MODE_INFO_EXT *mbmi_ext;
   int skip_block;
-  int select_tx_size;
   int q_index;
 
   // The equivalent error at the current rdmult of one whole bit (not one
diff --git a/av1/encoder/context_tree.c b/av1/encoder/context_tree.c
index c72e3038a..4f94571c4 100644
--- a/av1/encoder/context_tree.c
+++ b/av1/encoder/context_tree.c
@@ -20,40 +20,32 @@ static void alloc_mode_context(AV1_COMMON *cm, int num_4x4_blk,
                                PICK_MODE_CONTEXT *ctx) {
   const int num_blk = (num_4x4_blk < 4 ? 4 : num_4x4_blk);
   const int num_pix = num_blk << 4;
-  int i, k;
+  int i;
   ctx->num_4x4_blk = num_blk;
 
   for (i = 0; i < MAX_MB_PLANE; ++i) {
-    for (k = 0; k < 3; ++k) {
-      CHECK_MEM_ERROR(cm, ctx->coeff[i][k],
-                      aom_memalign(32, num_pix * sizeof(*ctx->coeff[i][k])));
-      CHECK_MEM_ERROR(cm, ctx->qcoeff[i][k],
-                      aom_memalign(32, num_pix * sizeof(*ctx->qcoeff[i][k])));
-      CHECK_MEM_ERROR(cm, ctx->dqcoeff[i][k],
-                      aom_memalign(32, num_pix * sizeof(*ctx->dqcoeff[i][k])));
-      CHECK_MEM_ERROR(cm, ctx->eobs[i][k],
-                      aom_memalign(32, num_blk * sizeof(*ctx->eobs[i][k])));
-      ctx->coeff_pbuf[i][k] = ctx->coeff[i][k];
-      ctx->qcoeff_pbuf[i][k] = ctx->qcoeff[i][k];
-      ctx->dqcoeff_pbuf[i][k] = ctx->dqcoeff[i][k];
-      ctx->eobs_pbuf[i][k] = ctx->eobs[i][k];
-    }
+    CHECK_MEM_ERROR(cm, ctx->coeff[i],
+                    aom_memalign(32, num_pix * sizeof(*ctx->coeff[i])));
+    CHECK_MEM_ERROR(cm, ctx->qcoeff[i],
+                    aom_memalign(32, num_pix * sizeof(*ctx->qcoeff[i])));
+    CHECK_MEM_ERROR(cm, ctx->dqcoeff[i],
+                    aom_memalign(32, num_pix * sizeof(*ctx->dqcoeff[i])));
+    CHECK_MEM_ERROR(cm, ctx->eobs[i],
+                    aom_memalign(32, num_blk * sizeof(*ctx->eobs[i])));
   }
 }
 
 static void free_mode_context(PICK_MODE_CONTEXT *ctx) {
-  int i, k;
+  int i;
   for (i = 0; i < MAX_MB_PLANE; ++i) {
-    for (k = 0; k < 3; ++k) {
-      aom_free(ctx->coeff[i][k]);
-      ctx->coeff[i][k] = 0;
-      aom_free(ctx->qcoeff[i][k]);
-      ctx->qcoeff[i][k] = 0;
-      aom_free(ctx->dqcoeff[i][k]);
-      ctx->dqcoeff[i][k] = 0;
-      aom_free(ctx->eobs[i][k]);
-      ctx->eobs[i][k] = 0;
-    }
+    aom_free(ctx->coeff[i]);
+    ctx->coeff[i] = 0;
+    aom_free(ctx->qcoeff[i]);
+    ctx->qcoeff[i] = 0;
+    aom_free(ctx->dqcoeff[i]);
+    ctx->dqcoeff[i] = 0;
+    aom_free(ctx->eobs[i]);
+    ctx->eobs[i] = 0;
   }
 
   for (i = 0; i < 2; ++i) {
diff --git a/av1/encoder/context_tree.h b/av1/encoder/context_tree.h
index 20c955ba4..f62df0170 100644
--- a/av1/encoder/context_tree.h
+++ b/av1/encoder/context_tree.h
@@ -28,16 +28,10 @@ typedef struct {
   MODE_INFO mic;
   MB_MODE_INFO_EXT mbmi_ext;
   uint8_t *color_index_map[2];
-  tran_low_t *coeff[MAX_MB_PLANE][3];
-  tran_low_t *qcoeff[MAX_MB_PLANE][3];
-  tran_low_t *dqcoeff[MAX_MB_PLANE][3];
-  uint16_t *eobs[MAX_MB_PLANE][3];
-
-  // dual buffer pointers, 0: in use, 1: best in store
-  tran_low_t *coeff_pbuf[MAX_MB_PLANE][3];
-  tran_low_t *qcoeff_pbuf[MAX_MB_PLANE][3];
-  tran_low_t *dqcoeff_pbuf[MAX_MB_PLANE][3];
-  uint16_t *eobs_pbuf[MAX_MB_PLANE][3];
+  tran_low_t *coeff[MAX_MB_PLANE];
+  tran_low_t *qcoeff[MAX_MB_PLANE];
+  tran_low_t *dqcoeff[MAX_MB_PLANE];
+  uint16_t *eobs[MAX_MB_PLANE];
 
   int num_4x4_blk;
   int skip;
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index a5a25cd8c..66f3bc9d1 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -901,7 +901,6 @@ static void update_state(const AV1_COMP *const cpi, ThreadData *td,
   const int mis = cm->mi_stride;
   const int mi_width = num_8x8_blocks_wide_lookup[bsize];
   const int mi_height = num_8x8_blocks_high_lookup[bsize];
-  int max_plane;
 #if CONFIG_REF_MV
   int8_t rf_type;
 #endif
@@ -945,19 +944,11 @@ static void update_state(const AV1_COMP *const cpi, ThreadData *td,
     }
   }
 
-  max_plane = is_inter_block(mbmi) ? MAX_MB_PLANE : 1;
-  for (i = 0; i < max_plane; ++i) {
-    p[i].coeff = ctx->coeff_pbuf[i][1];
-    p[i].qcoeff = ctx->qcoeff_pbuf[i][1];
-    pd[i].dqcoeff = ctx->dqcoeff_pbuf[i][1];
-    p[i].eobs = ctx->eobs_pbuf[i][1];
-  }
-
-  for (i = max_plane; i < MAX_MB_PLANE; ++i) {
-    p[i].coeff = ctx->coeff_pbuf[i][2];
-    p[i].qcoeff = ctx->qcoeff_pbuf[i][2];
-    pd[i].dqcoeff = ctx->dqcoeff_pbuf[i][2];
-    p[i].eobs = ctx->eobs_pbuf[i][2];
+  for (i = 0; i < MAX_MB_PLANE; ++i) {
+    p[i].coeff = ctx->coeff[i];
+    p[i].qcoeff = ctx->qcoeff[i];
+    pd[i].dqcoeff = ctx->dqcoeff[i];
+    p[i].eobs = ctx->eobs[i];
   }
 
   for (i = 0; i < 2; ++i) pd[i].color_index_map = ctx->color_index_map[i];
@@ -1090,10 +1081,10 @@ static void rd_pick_sb_modes(const AV1_COMP *const cpi, TileDataEnc *tile_data,
   mbmi->sb_type = bsize;
 
   for (i = 0; i < MAX_MB_PLANE; ++i) {
-    p[i].coeff = ctx->coeff_pbuf[i][0];
-    p[i].qcoeff = ctx->qcoeff_pbuf[i][0];
-    pd[i].dqcoeff = ctx->dqcoeff_pbuf[i][0];
-    p[i].eobs = ctx->eobs_pbuf[i][0];
+    p[i].coeff = ctx->coeff[i];
+    p[i].qcoeff = ctx->qcoeff[i];
+    pd[i].dqcoeff = ctx->dqcoeff[i];
+    p[i].eobs = ctx->eobs[i];
   }
 
   for (i = 0; i < 2; ++i) pd[i].color_index_map = ctx->color_index_map[i];
diff --git a/av1/encoder/firstpass.c b/av1/encoder/firstpass.c
index bca4868a6..1f5aef7f3 100644
--- a/av1/encoder/firstpass.c
+++ b/av1/encoder/firstpass.c
@@ -520,10 +520,10 @@ void av1_first_pass(AV1_COMP *cpi, const struct lookahead_entry *source) {
   av1_frame_init_quantizer(cpi);
 
   for (i = 0; i < MAX_MB_PLANE; ++i) {
-    p[i].coeff = ctx->coeff_pbuf[i][1];
-    p[i].qcoeff = ctx->qcoeff_pbuf[i][1];
-    pd[i].dqcoeff = ctx->dqcoeff_pbuf[i][1];
-    p[i].eobs = ctx->eobs_pbuf[i][1];
+    p[i].coeff = ctx->coeff[i];
+    p[i].qcoeff = ctx->qcoeff[i];
+    pd[i].dqcoeff = ctx->dqcoeff[i];
+    p[i].eobs = ctx->eobs[i];
   }
 
   av1_init_mv_probs(cm);
diff --git a/av1/encoder/rd.c b/av1/encoder/rd.c
index 9084cd810..8dd69c2ed 100644
--- a/av1/encoder/rd.c
+++ b/av1/encoder/rd.c
@@ -290,11 +290,6 @@ void av1_initialize_rd_consts(AV1_COMP *cpi) {
 
   set_error_per_bit(x, rd->RDMULT);
 
-  x->select_tx_size = (cpi->sf.tx_size_search_method == USE_LARGESTALL &&
-                       cm->frame_type != KEY_FRAME)
-                          ? 0
-                          : 1;
-
   set_block_thresholds(cm, rd);
 
   fill_token_costs(x->token_costs, cm->fc->coef_probs);
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 3346d37ae..c08a16eb8 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -233,31 +233,6 @@ static const REF_DEFINITION av1_ref_order[MAX_REFS] = {
   { { INTRA_FRAME, NONE } },
 };
 
-static void swap_block_ptr(MACROBLOCK *x, PICK_MODE_CONTEXT *ctx, int m, int n,
-                           int min_plane, int max_plane) {
-  int i;
-
-  for (i = min_plane; i < max_plane; ++i) {
-    struct macroblock_plane *const p = &x->plane[i];
-    struct macroblockd_plane *const pd = &x->e_mbd.plane[i];
-
-    p->coeff = ctx->coeff_pbuf[i][m];
-    p->qcoeff = ctx->qcoeff_pbuf[i][m];
-    pd->dqcoeff = ctx->dqcoeff_pbuf[i][m];
-    p->eobs = ctx->eobs_pbuf[i][m];
-
-    ctx->coeff_pbuf[i][m] = ctx->coeff_pbuf[i][n];
-    ctx->qcoeff_pbuf[i][m] = ctx->qcoeff_pbuf[i][n];
-    ctx->dqcoeff_pbuf[i][m] = ctx->dqcoeff_pbuf[i][n];
-    ctx->eobs_pbuf[i][m] = ctx->eobs_pbuf[i][n];
-
-    ctx->coeff_pbuf[i][n] = p->coeff;
-    ctx->qcoeff_pbuf[i][n] = p->qcoeff;
-    ctx->dqcoeff_pbuf[i][n] = pd->dqcoeff;
-    ctx->eobs_pbuf[i][n] = p->eobs;
-  }
-}
-
 static void model_rd_for_sb(const AV1_COMP *const cpi, BLOCK_SIZE bsize,
                             MACROBLOCK *x, MACROBLOCKD *xd, int *out_rate_sum,
                             int64_t *out_dist_sum, int *skip_txfm_sb,
@@ -270,11 +245,9 @@ static void model_rd_for_sb(const AV1_COMP *const cpi, BLOCK_SIZE bsize,
   int64_t dist_sum = 0;
   const int ref = xd->mi[0]->mbmi.ref_frame[0];
   unsigned int sse;
-  unsigned int var = 0;
   unsigned int sum_sse = 0;
   int64_t total_sse = 0;
   int skip_flag = 1;
-  const int shift = 6;
   int rate;
   int64_t dist;
   const int dequant_shift =
@@ -291,12 +264,6 @@ static void model_rd_for_sb(const AV1_COMP *const cpi, BLOCK_SIZE bsize,
     const BLOCK_SIZE bs = get_plane_block_size(bsize, pd);
     const TX_SIZE max_tx_size = max_txsize_lookup[bs];
     const BLOCK_SIZE unit_size = txsize_to_bsize[max_tx_size];
-    const int64_t dc_thr = p->quant_thred[0] >> shift;
-    const int64_t ac_thr = p->quant_thred[1] >> shift;
-    // The low thresholds are used to measure if the prediction errors are
-    // low enough so that we can skip the mode search.
-    const int64_t low_dc_thr = AOMMIN(50, dc_thr >> 2);
-    const int64_t low_ac_thr = AOMMIN(80, ac_thr >> 2);
     int bw = 1 << (b_width_log2_lookup[bs] - b_width_log2_lookup[unit_size]);
     int bh = 1 << (b_height_log2_lookup[bs] - b_width_log2_lookup[unit_size]);
     int idx, idy;
@@ -312,26 +279,12 @@ static void model_rd_for_sb(const AV1_COMP *const cpi, BLOCK_SIZE bsize,
         int block_idx = (idy << 1) + idx;
         int low_err_skip = 0;
 
-        var = cpi->fn_ptr[unit_size].vf(src, p->src.stride, dst, pd->dst.stride,
-                                        &sse);
+        cpi->fn_ptr[unit_size].vf(src, p->src.stride, dst, pd->dst.stride,
+                                  &sse);
         x->bsse[(i << 2) + block_idx] = sse;
         sum_sse += sse;
 
         x->skip_txfm[(i << 2) + block_idx] = SKIP_TXFM_NONE;
-        if (!x->select_tx_size) {
-          // Check if all ac coefficients can be quantized to zero.
-          if (var < ac_thr || var == 0) {
-            x->skip_txfm[(i << 2) + block_idx] = SKIP_TXFM_AC_ONLY;
-
-            // Check if dc coefficient can be quantized to zero.
-            if (sse - var < dc_thr || sse == var) {
-              x->skip_txfm[(i << 2) + block_idx] = SKIP_TXFM_AC_DC;
-
-              if (!sse || (var < low_ac_thr && sse - var < low_dc_thr))
-                low_err_skip = 1;
-            }
-          }
-        }
 
         if (skip_flag && !low_err_skip) skip_flag = 0;
 
@@ -1676,10 +1629,9 @@ static int rd_pick_intra_angle_sbuv(const AV1_COMP *cpi, MACROBLOCK *x,
 #endif  // CONFIG_EXT_INTRA
 
 static int64_t rd_pick_intra_sbuv_mode(const AV1_COMP *const cpi, MACROBLOCK *x,
-                                       PICK_MODE_CONTEXT *ctx, int *rate,
-                                       int *rate_tokenonly, int64_t *distortion,
-                                       int *skippable, BLOCK_SIZE bsize,
-                                       TX_SIZE max_tx_size) {
+                                       int *rate, int *rate_tokenonly,
+                                       int64_t *distortion, int *skippable,
+                                       BLOCK_SIZE bsize, TX_SIZE max_tx_size) {
   MB_MODE_INFO *const mbmi = &x->e_mbd.mi[0]->mbmi;
   PREDICTION_MODE mode;
   PREDICTION_MODE mode_selected = DC_PRED;
@@ -1735,7 +1687,6 @@ static int64_t rd_pick_intra_sbuv_mode(const AV1_COMP *const cpi, MACROBLOCK *x,
       *rate_tokenonly = this_rate_tokenonly;
       *distortion = this_distortion;
       *skippable = s;
-      if (!x->select_tx_size) swap_block_ptr(x, ctx, 2, 0, 1, MAX_MB_PLANE);
     }
   }
 
@@ -1747,15 +1698,14 @@ static int64_t rd_pick_intra_sbuv_mode(const AV1_COMP *const cpi, MACROBLOCK *x,
 }
 
 static void choose_intra_uv_mode(const AV1_COMP *const cpi, MACROBLOCK *const x,
-                                 PICK_MODE_CONTEXT *ctx, BLOCK_SIZE bsize,
-                                 TX_SIZE max_tx_size, int *rate_uv,
-                                 int *rate_uv_tokenonly, int64_t *dist_uv,
-                                 int *skip_uv, PREDICTION_MODE *mode_uv) {
+                                 BLOCK_SIZE bsize, TX_SIZE max_tx_size,
+                                 int *rate_uv, int *rate_uv_tokenonly,
+                                 int64_t *dist_uv, int *skip_uv,
+                                 PREDICTION_MODE *mode_uv) {
   // Use an estimated rd for uv_intra based on DC_PRED if the
   // appropriate speed flag is set.
-  rd_pick_intra_sbuv_mode(cpi, x, ctx, rate_uv, rate_uv_tokenonly, dist_uv,
-                          skip_uv, bsize < BLOCK_8X8 ? BLOCK_8X8 : bsize,
-                          max_tx_size);
+  rd_pick_intra_sbuv_mode(cpi, x, rate_uv, rate_uv_tokenonly, dist_uv, skip_uv,
+                          bsize < BLOCK_8X8 ? BLOCK_8X8 : bsize, max_tx_size);
   *mode_uv = x->e_mbd.mi[0]->mbmi.uv_mode;
 }
 
@@ -3591,7 +3541,7 @@ void av1_rd_pick_intra_mode_sb(const AV1_COMP *cpi, MACROBLOCK *x,
   }
   max_uv_tx_size = get_uv_tx_size_impl(
       xd->mi[0]->mbmi.tx_size, bsize, pd[1].subsampling_x, pd[1].subsampling_y);
-  rd_pick_intra_sbuv_mode(cpi, x, ctx, &rate_uv, &rate_uv_tokenonly, &dist_uv,
+  rd_pick_intra_sbuv_mode(cpi, x, &rate_uv, &rate_uv_tokenonly, &dist_uv,
                           &uv_skip, AOMMAX(BLOCK_8X8, bsize), max_uv_tx_size);
 
   if (y_skip && uv_skip) {
@@ -4206,7 +4156,7 @@ void av1_rd_pick_inter_mode_sb(const AV1_COMP *cpi, TileDataEnc *tile_data,
       uv_tx = get_uv_tx_size_impl(mbmi->tx_size, bsize, pd->subsampling_x,
                                   pd->subsampling_y);
       if (rate_uv_intra[uv_tx] == INT_MAX) {
-        choose_intra_uv_mode(cpi, x, ctx, bsize, uv_tx, &rate_uv_intra[uv_tx],
+        choose_intra_uv_mode(cpi, x, bsize, uv_tx, &rate_uv_intra[uv_tx],
                              &rate_uv_tokenonly[uv_tx], &dist_uv[uv_tx],
                              &skip_uv[uv_tx], &mode_uv[uv_tx]);
 #if CONFIG_EXT_INTRA
@@ -4482,7 +4432,6 @@ void av1_rd_pick_inter_mode_sb(const AV1_COMP *cpi, TileDataEnc *tile_data,
 
     // Did this mode help.. i.e. is it the new best mode
     if (this_rd < best_rd || x->skip) {
-      int max_plane = MAX_MB_PLANE;
       if (!mode_excluded) {
         // Note index of best mode so far
         best_mode_index = mode_index;
@@ -4490,7 +4439,6 @@ void av1_rd_pick_inter_mode_sb(const AV1_COMP *cpi, TileDataEnc *tile_data,
         if (ref_frame == INTRA_FRAME) {
           /* required for left and above block mv */
           mbmi->mv[0].as_int = 0;
-          max_plane = 1;
         } else {
           best_pred_sse = x->pred_sse[ref_frame];
         }
@@ -4502,8 +4450,6 @@ void av1_rd_pick_inter_mode_sb(const AV1_COMP *cpi, TileDataEnc *tile_data,
         best_mbmode = *mbmi;
         best_skip2 = this_skip2;
         best_mode_skippable = skippable;
-
-        if (!x->select_tx_size) swap_block_ptr(x, ctx, 1, 0, 0, max_plane);
       }
     }
 
@@ -4668,23 +4614,6 @@ void av1_rd_pick_inter_mode_sb(const AV1_COMP *cpi, TileDataEnc *tile_data,
 
   x->skip |= best_mode_skippable;
 
-  if (!x->skip && !x->select_tx_size) {
-    int has_high_freq_coeff = 0;
-    int plane;
-    int max_plane = is_inter_block(&xd->mi[0]->mbmi) ? MAX_MB_PLANE : 1;
-    for (plane = 0; plane < max_plane; ++plane) {
-      x->plane[plane].eobs = ctx->eobs_pbuf[plane][1];
-      has_high_freq_coeff |= av1_has_high_freq_in_plane(x, bsize, plane);
-    }
-
-    for (plane = max_plane; plane < MAX_MB_PLANE; ++plane) {
-      x->plane[plane].eobs = ctx->eobs_pbuf[plane][2];
-      has_high_freq_coeff |= av1_has_high_freq_in_plane(x, bsize, plane);
-    }
-
-    best_mode_skippable |= !has_high_freq_coeff;
-  }
-
   assert(best_mode_index >= 0);
 
   store_coding_context(x, ctx, best_mode_index, best_pred_diff,
@@ -4790,7 +4719,6 @@ void av1_rd_pick_inter_mode_sb_seg_skip(const AV1_COMP *cpi,
 
   av1_zero(best_pred_diff);
 
-  if (!x->select_tx_size) swap_block_ptr(x, ctx, 1, 0, 0, MAX_MB_PLANE);
   store_coding_context(x, ctx, THR_ZEROMV, best_pred_diff, 0);
 }
 
@@ -5046,7 +4974,7 @@ void av1_rd_pick_inter_mode_sub8x8(const AV1_COMP *cpi, TileDataEnc *tile_data,
       distortion2 += distortion_y;
 
       if (rate_uv_intra == INT_MAX) {
-        choose_intra_uv_mode(cpi, x, ctx, bsize, TX_4X4, &rate_uv_intra,
+        choose_intra_uv_mode(cpi, x, bsize, TX_4X4, &rate_uv_intra,
                              &rate_uv_tokenonly, &dist_uv, &skip_uv, &mode_uv);
       }
       rate2 += rate_uv_intra;
@@ -5261,14 +5189,12 @@ void av1_rd_pick_inter_mode_sub8x8(const AV1_COMP *cpi, TileDataEnc *tile_data,
     // Did this mode help.. i.e. is it the new best mode
     if (this_rd < best_rd || x->skip) {
       if (!mode_excluded) {
-        int max_plane = MAX_MB_PLANE;
         // Note index of best mode so far
         best_ref_index = ref_index;
 
         if (ref_frame == INTRA_FRAME) {
           /* required for left and above block mv */
           mbmi->mv[0].as_int = 0;
-          max_plane = 1;
         }
 
         rd_cost->rate = rate2;
@@ -5279,7 +5205,6 @@ void av1_rd_pick_inter_mode_sub8x8(const AV1_COMP *cpi, TileDataEnc *tile_data,
             best_rd - RDCOST(x->rdmult, x->rddiv, rate_uv, distortion_uv);
         best_mbmode = *mbmi;
         best_skip2 = this_skip2;
-        if (!x->select_tx_size) swap_block_ptr(x, ctx, 1, 0, 0, max_plane);
 
         for (i = 0; i < 4; i++) best_bmodes[i] = xd->mi[0]->bmi[i];
       }