Use hybrid RD and non-RD coding flow for key frame coding

When block size is below 16x16, the encoder swap from non-RD to
RD mode for key frame coding. This largely brough back the key
frame compression performance. For vidyo1 at 1000 kbps, the key
frame coding statistics are changed

9978F, 34.183 dB, 36807 us -> 9838F, 35.020 dB, 61677 us

As compared to the full RD case
7187F, 34.930 dB, 214470 us

The overall rtc set coding performance (single key frame setting)
is improved by 1.5%.

Change-Id: I78a4ecf025d7b24ec911e85be94e01da05e77878

vp9/encoder/vp9_encodeframe.c

@@ -2662,6 +2662,15 @@ static TX_MODE select_tx_mode(const VP9_COMP *cpi, MACROBLOCKD *const xd) {
     return cpi->common.tx_mode;
 }
 
+static void hybrid_intra_mode_search(VP9_COMP *cpi, MACROBLOCK *const x,
+                                     RD_COST *rd_cost, BLOCK_SIZE bsize,
+                                     PICK_MODE_CONTEXT *ctx) {
+  if (bsize < BLOCK_16X16)
+    vp9_rd_pick_intra_mode_sb(cpi, x, rd_cost, bsize, ctx, INT64_MAX);
+  else
+    vp9_pick_intra_mode(cpi, x, rd_cost, bsize, ctx);
+}
+
 static void nonrd_pick_sb_modes(VP9_COMP *cpi,
                                 TileDataEnc *tile_data, MACROBLOCK *const x,
                                 int mi_row, int mi_col, RD_COST *rd_cost,
@@ -2679,7 +2688,7 @@ static void nonrd_pick_sb_modes(VP9_COMP *cpi,
       x->rdmult = vp9_cyclic_refresh_get_rdmult(cpi->cyclic_refresh);
 
   if (cm->frame_type == KEY_FRAME)
-    vp9_pick_intra_mode(cpi, x, rd_cost, bsize, ctx);
+    hybrid_intra_mode_search(cpi, x, rd_cost, bsize, ctx);
   else if (vp9_segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP))
     set_mode_info_seg_skip(x, cm->tx_mode, rd_cost, bsize);
   else
@@ -3604,6 +3613,7 @@ static void encode_frame_internal(VP9_COMP *cpi) {
     cm->tx_mode = ALLOW_16X16;
   }
 
+
 #if CONFIG_VP9_HIGHBITDEPTH
   if (cm->use_highbitdepth)
     x->fwd_txm4x4 = xd->lossless ? vp9_highbd_fwht4x4 : vp9_highbd_fdct4x4;

vp9/encoder/vp9_speed_features.c

@@ -321,7 +321,6 @@ static void set_rt_speed_feature(VP9_COMP *cpi, SPEED_FEATURES *sf,
   if (speed >= 6) {
     // Adaptively switch between SOURCE_VAR_BASED_PARTITION and FIXED_PARTITION.
     sf->partition_search_type = VAR_BASED_PARTITION;
-
     // Turn on this to use non-RD key frame coding mode.
     sf->use_nonrd_pick_mode = 1;
     sf->mv.search_method = NSTEP;