Enable reduced set of intra modes in rtc coding

This commit enables the use of DC, vertical, and horizontal intra
prediction mode in rtc non-RD mode decision. When the best cost value
of inter modes is above a given threshold, the encoder runs the
above three intra modes and selects the one that has minimum
prediction residual in terms of SAD.

This together with recent changes on non-RD mode decision and coding
control improves compression performance of speed -6 by
derf  91%
yt    61%
hd    46%
stdhd 52%

In terms of encoding speed, it is about 3 times faster than speed -5.

Change-Id: I6b483bfd0307e6482bb22a6676ae4e25a52b1310

vp9/encoder/vp9_pickmode.c

@@ -194,6 +194,9 @@ int64_t vp9_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
   int64_t this_rd;
   int64_t cost[4]= { 0, 50, 75, 100 };
 
+  const int64_t inter_mode_thresh = 300;
+  const int64_t intra_mode_cost = 50;
+
   x->skip_encode = cpi->sf.skip_encode_frame && x->q_index < QIDX_SKIP_THRESH;
 
   x->skip = 0;
@@ -264,6 +267,31 @@ int64_t vp9_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
     }
   }
 
+  // Perform intra prediction search, if the best SAD is above a certain
+  // threshold.
+  if (best_rd > inter_mode_thresh) {
+    struct macroblock_plane *const p = &x->plane[0];
+    struct macroblockd_plane *const pd = &xd->plane[0];
+    for (this_mode = DC_PRED; this_mode <= H_PRED; ++this_mode) {
+      vp9_predict_intra_block(xd, 0, b_width_log2(bsize),
+                              mbmi->tx_size, this_mode,
+                              &p->src.buf[0], p->src.stride,
+                              &pd->dst.buf[0], pd->dst.stride, 0, 0, 0);
+
+      this_rd = cpi->fn_ptr[bsize].sdf(p->src.buf,
+                                       p->src.stride,
+                                       pd->dst.buf,
+                                       pd->dst.stride, INT_MAX);
+
+      if (this_rd + intra_mode_cost < best_rd) {
+        best_rd = this_rd;
+        mbmi->mode = this_mode;
+        mbmi->ref_frame[0] = INTRA_FRAME;
+        mbmi->uv_mode = this_mode;
+      }
+    }
+  }
+
   // Perform sub-pixel motion search, if NEWMV is chosen
   if (mbmi->mode == NEWMV) {
     ref_frame = mbmi->ref_frame[0];
@@ -273,8 +301,5 @@ int64_t vp9_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
     xd->mi_8x8[0]->bmi[0].as_mv[0].as_int = mbmi->mv[0].as_int;
   }
 
-  // TODO(jingning) intra prediction search, if the best SAD is above a certain
-  // threshold.
-
   return INT64_MAX;
 }