Add encoder first pass support to bi-prediction in EXT_REFS

For the experiment of EXT_REFS, 3 new frame update types are added,
namely
(1) BRF_UPDATE: The frame served as BWDREF_FRAME;
(2) LAST_BIPRED_UPDATE: The last frame within the BWDREF_FRAME group
    that is being bi-directionally predicted using BWDREF_FRAME; and
(3) BIPRED_UPDATE: The frame, except the last one, within the
    BWDREF_FRAME group that is being bi-directionally predicted using
    BWDREF_FRAME.

Change-Id: I64f6ba3f45d6a3dcd1ee97567c6f904ad4e7bad3

av1/common/odintrin.h

@@ -11,10 +11,10 @@
 #ifndef AV1_COMMON_ODINTRIN_H_
 #define AV1_COMMON_ODINTRIN_H_
 
-#include "av1/common/enums.h"
 #include "aom/aom_integer.h"
 #include "aom_dsp/aom_dsp_common.h"
 #include "aom_ports/bitops.h"
+#include "av1/common/enums.h"
 
 #ifdef __cplusplus
 extern "C" {

av1/encoder/encoder.h

@@ -324,6 +324,9 @@ typedef struct AV1_COMP {
 
   int refresh_last_frame;
   int refresh_golden_frame;
+#if CONFIG_EXT_REFS
+  int refresh_bwd_ref_frame;
+#endif  // CONFIG_EXT_REFS
   int refresh_alt_ref_frame;
 
   int ext_refresh_frame_flags_pending;

av1/encoder/firstpass.c

@@ -23,6 +23,7 @@
 #include "aom_scale/aom_scale.h"
 #include "aom_scale/yv12config.h"
 
+#include "aom_dsp/variance.h"
 #include "av1/common/entropymv.h"
 #include "av1/common/quant_common.h"
 #include "av1/common/reconinter.h"  // av1_setup_dst_planes()
@@ -37,7 +38,6 @@
 #include "av1/encoder/mcomp.h"
 #include "av1/encoder/quantize.h"
 #include "av1/encoder/rd.h"
-#include "aom_dsp/variance.h"
 
 #define OUTPUT_FPF 0
 #define ARF_STATS_OUTPUT 0
@@ -1536,7 +1536,7 @@ static void allocate_gf_group_bits(AV1_COMP *cpi, int64_t gf_group_bits,
   GF_GROUP *const gf_group = &twopass->gf_group;
   FIRSTPASS_STATS frame_stats;
   int i;
-  int frame_index = 1;
+  int frame_index = 0;
   int target_frame_size;
   int key_frame;
   const int max_bits = frame_max_bits(&cpi->rc, &cpi->oxcf);
@@ -1546,7 +1546,21 @@ static void allocate_gf_group_bits(AV1_COMP *cpi, int64_t gf_group_bits,
   int mid_boost_bits = 0;
   int mid_frame_idx;
   unsigned char arf_buffer_indices[MAX_ACTIVE_ARFS];
-  int alt_frame_index = frame_index;
+
+#if CONFIG_EXT_REFS
+  // The use of bi-predictive frames are only enabled when following 3
+  // conditions are met:
+  // (1) Alt-ref is enabled;
+  // (2) The bi-predictive group interval is at least 2; and
+  // (3) The bi-predictive group interval is strictly smaller than the
+  //     golden group interval.
+  const int is_bipred_enabled =
+      rc->source_alt_ref_pending && rc->bipred_group_interval &&
+      rc->bipred_group_interval <=
+          (rc->baseline_gf_interval - rc->source_alt_ref_pending);
+  int bipred_group_end = 0;
+  int bipred_frame_index = 0;
+#endif  // CONFIG_EXT_REFS
 
   key_frame = cpi->common.frame_type == KEY_FRAME;
 
@@ -1556,40 +1570,54 @@ static void allocate_gf_group_bits(AV1_COMP *cpi, int64_t gf_group_bits,
   // is also the golden frame.
   if (!key_frame) {
     if (rc->source_alt_ref_active) {
-      gf_group->update_type[0] = OVERLAY_UPDATE;
-      gf_group->rf_level[0] = INTER_NORMAL;
-      gf_group->bit_allocation[0] = 0;
-      gf_group->arf_update_idx[0] = arf_buffer_indices[0];
-      gf_group->arf_ref_idx[0] = arf_buffer_indices[0];
+      gf_group->update_type[frame_index] = OVERLAY_UPDATE;
+      gf_group->rf_level[frame_index] = INTER_NORMAL;
+      gf_group->bit_allocation[frame_index] = 0;
     } else {
-      gf_group->update_type[0] = GF_UPDATE;
-      gf_group->rf_level[0] = GF_ARF_STD;
-      gf_group->bit_allocation[0] = gf_arf_bits;
-      gf_group->arf_update_idx[0] = arf_buffer_indices[0];
-      gf_group->arf_ref_idx[0] = arf_buffer_indices[0];
+      gf_group->update_type[frame_index] = GF_UPDATE;
+      gf_group->rf_level[frame_index] = GF_ARF_STD;
+      gf_group->bit_allocation[frame_index] = gf_arf_bits;
     }
+    gf_group->arf_update_idx[frame_index] = arf_buffer_indices[0];
+    gf_group->arf_ref_idx[frame_index] = arf_buffer_indices[0];
 
     // Step over the golden frame / overlay frame
     if (EOF == input_stats(twopass, &frame_stats)) return;
   }
 
+#if CONFIG_EXT_REFS
+  gf_group->brf_src_offset[frame_index] = 0;
+  gf_group->brf_pred_enabled[frame_index] = 0;
+#endif  // CONFIG_EXT_REFS
+
   // Deduct the boost bits for arf (or gf if it is not a key frame)
   // from the group total.
   if (rc->source_alt_ref_pending || !key_frame) total_group_bits -= gf_arf_bits;
 
+  frame_index++;
+#if CONFIG_EXT_REFS
+  bipred_frame_index++;
+#endif  // CONFIG_EXT_REFS
+
   // Store the bits to spend on the ARF if there is one.
   if (rc->source_alt_ref_pending) {
-    gf_group->update_type[alt_frame_index] = ARF_UPDATE;
-    gf_group->rf_level[alt_frame_index] = GF_ARF_STD;
-    gf_group->bit_allocation[alt_frame_index] = gf_arf_bits;
+    gf_group->update_type[frame_index] = ARF_UPDATE;
+    gf_group->rf_level[frame_index] = GF_ARF_STD;
+    gf_group->bit_allocation[frame_index] = gf_arf_bits;
 
-    gf_group->arf_src_offset[alt_frame_index] =
+    gf_group->arf_src_offset[frame_index] =
         (unsigned char)(rc->baseline_gf_interval - 1);
 
-    gf_group->arf_update_idx[alt_frame_index] = arf_buffer_indices[0];
-    gf_group->arf_ref_idx[alt_frame_index] =
+    gf_group->arf_update_idx[frame_index] = arf_buffer_indices[0];
+    gf_group->arf_ref_idx[frame_index] =
         arf_buffer_indices[cpi->multi_arf_last_grp_enabled &&
                            rc->source_alt_ref_active];
+#if CONFIG_EXT_REFS
+    gf_group->brf_src_offset[frame_index] = 0;
+    gf_group->brf_pred_enabled[frame_index] = 0;
+// NOTE: "bipred_frame_index" stays unchanged for ARF_UPDATE frames.
+#endif  // CONFIG_EXT_REFS
+
     ++frame_index;
 
     if (cpi->multi_arf_enabled) {
@@ -1633,10 +1661,73 @@ static void allocate_gf_group_bits(AV1_COMP *cpi, int64_t gf_group_bits,
     target_frame_size =
         clamp(target_frame_size, 0, AOMMIN(max_bits, (int)total_group_bits));
 
-    gf_group->update_type[frame_index] = LF_UPDATE;
-    gf_group->rf_level[frame_index] = INTER_NORMAL;
+#if CONFIG_EXT_REFS
+    // NOTE: Bi-predictive frames are only enabled when the length of the
+    //       bi-predictive frame group interval is strictly smaller than that
+    //       of the golden frame group interval.
+    // TODO(zoeliu): Currently bi-prediction is only enabled when alt-ref is on.
+    if (is_bipred_enabled && !bipred_group_end) {
+      const int cur_brf_src_offset = rc->bipred_group_interval - 1;
 
+      // --- BRF_UPDATE ---
+      if (bipred_frame_index == 1) {
+        gf_group->update_type[frame_index] = BRF_UPDATE;
+        gf_group->brf_src_offset[frame_index] = cur_brf_src_offset;
+        gf_group->brf_pred_enabled[frame_index] = 1;
+        // --- LAST_BIPRED_UPDATE ---
+      } else if (bipred_frame_index == rc->bipred_group_interval) {
+        gf_group->update_type[frame_index] = LAST_BIPRED_UPDATE;
+        gf_group->brf_src_offset[frame_index] = 0;
+        gf_group->brf_pred_enabled[frame_index] = 1;
+        // Reset the bi-predictive frame index.
+        bipred_frame_index = 0;
+        // --- BIPRED_UPDATE ---
+      } else {
+        gf_group->update_type[frame_index] = BIPRED_UPDATE;
+        gf_group->brf_src_offset[frame_index] = 0;
+        gf_group->brf_pred_enabled[frame_index] = 1;
+      }
+
+      bipred_frame_index++;
+      // Check whether the next bi-predictive frame group would entirely be
+      // included within the current golden frame group.
+      if (bipred_frame_index == 1 &&
+          (i + 1 + cur_brf_src_offset) >=
+              (rc->baseline_gf_interval - rc->source_alt_ref_pending))
+        bipred_group_end = 1;
+    } else {
+#endif  // CONFIG_EXT_REFS
+      gf_group->update_type[frame_index] = LF_UPDATE;
+#if CONFIG_EXT_REFS
+      gf_group->brf_src_offset[frame_index] = 0;
+      gf_group->brf_pred_enabled[frame_index] = 0;
+    }
+#endif  // CONFIG_EXT_REFS
+
+#if CONFIG_EXT_REFS
+    if (gf_group->update_type[frame_index] == BRF_UPDATE) {
+      // Boost up the allocated bits on BWDREF_FRAME
+      gf_group->rf_level[frame_index] = INTER_HIGH;
+      gf_group->bit_allocation[frame_index] =
+          target_frame_size + (target_frame_size >> 2);
+    } else if (gf_group->update_type[frame_index] == LAST_BIPRED_UPDATE) {
+      // Press down the allocated bits on LAST_BIPRED_UPDATE frames
+      gf_group->rf_level[frame_index] = INTER_LOW;
+      gf_group->bit_allocation[frame_index] =
+          target_frame_size - (target_frame_size >> 1);
+    } else if (gf_group->update_type[frame_index] == BIPRED_UPDATE) {
+      // TODO(zoeliu): To investigate whether the allocated bits on
+      // BIPRED_UPDATE frames need to be further adjusted.
+      gf_group->rf_level[frame_index] = INTER_NORMAL;
       gf_group->bit_allocation[frame_index] = target_frame_size;
+    } else {
+#endif  // CONFIG_EXT_REFS
+      gf_group->rf_level[frame_index] = INTER_NORMAL;
+      gf_group->bit_allocation[frame_index] = target_frame_size;
+#if CONFIG_EXT_REFS
+    }
+#endif  // CONFIG_EXT_REFS
+
     ++frame_index;
   }
 
@@ -1662,6 +1753,10 @@ static void allocate_gf_group_bits(AV1_COMP *cpi, int64_t gf_group_bits,
     gf_group->update_type[frame_index] = GF_UPDATE;
     gf_group->rf_level[frame_index] = GF_ARF_STD;
   }
+#if CONFIG_EXT_REFS
+  gf_group->brf_src_offset[frame_index] = 0;
+  gf_group->brf_pred_enabled[frame_index] = 0;
+#endif  // CONFIG_EXT_REFS
 
   // Note whether multi-arf was enabled this group for next time.
   cpi->multi_arf_last_grp_enabled = cpi->multi_arf_enabled;
@@ -1871,6 +1966,13 @@ static void define_gf_group(AV1_COMP *cpi, FIRSTPASS_STATS *this_frame) {
 
   rc->frames_till_gf_update_due = rc->baseline_gf_interval;
 
+#if CONFIG_EXT_REFS
+  rc->bipred_group_interval = BFG_INTERVAL;
+  // The minimum bi-predictive frame group interval is 2.
+  if (rc->bipred_group_interval < MIN_BFG_INTERVAL)
+    rc->bipred_group_interval = 0;
+#endif  // CONFIG_EXT_REFS
+
   // Reset the file position.
   reset_fpf_position(twopass, start_pos);
 
@@ -2295,33 +2397,69 @@ static void configure_buffer_updates(AV1_COMP *cpi) {
   TWO_PASS *const twopass = &cpi->twopass;
 
   cpi->rc.is_src_frame_alt_ref = 0;
+#if CONFIG_EXT_REFS
+  cpi->rc.is_bwd_ref_frame = 0;
+  cpi->rc.is_last_bipred_frame = 0;
+  cpi->rc.is_bipred_frame = 0;
+#endif  // CONFIG_EXT_REFS
+
   switch (twopass->gf_group.update_type[twopass->gf_group.index]) {
-    case KF_UPDATE:
-      cpi->refresh_last_frame = 1;
-      cpi->refresh_golden_frame = 1;
+    case KF_UPDATE: cpi->refresh_last_frame = 1; cpi->refresh_golden_frame = 1;
+#if CONFIG_EXT_REFS
+      cpi->refresh_bwd_ref_frame = 1;
+#endif  // CONFIG_EXT_REFS
       cpi->refresh_alt_ref_frame = 1;
       break;
-    case LF_UPDATE:
-      cpi->refresh_last_frame = 1;
-      cpi->refresh_golden_frame = 0;
+    case LF_UPDATE: cpi->refresh_last_frame = 1; cpi->refresh_golden_frame = 0;
+#if CONFIG_EXT_REFS
+      cpi->refresh_bwd_ref_frame = 0;
+#endif  // CONFIG_EXT_REFS
       cpi->refresh_alt_ref_frame = 0;
       break;
-    case GF_UPDATE:
-      cpi->refresh_last_frame = 1;
-      cpi->refresh_golden_frame = 1;
+    case GF_UPDATE: cpi->refresh_last_frame = 1; cpi->refresh_golden_frame = 1;
+#if CONFIG_EXT_REFS
+      cpi->refresh_bwd_ref_frame = 0;
+#endif  // CONFIG_EXT_REFS
       cpi->refresh_alt_ref_frame = 0;
       break;
     case OVERLAY_UPDATE:
       cpi->refresh_last_frame = 0;
       cpi->refresh_golden_frame = 1;
+#if CONFIG_EXT_REFS
+      cpi->refresh_bwd_ref_frame = 0;
+#endif  // CONFIG_EXT_REFS
       cpi->refresh_alt_ref_frame = 0;
       cpi->rc.is_src_frame_alt_ref = 1;
       break;
-    case ARF_UPDATE:
-      cpi->refresh_last_frame = 0;
-      cpi->refresh_golden_frame = 0;
+    case ARF_UPDATE: cpi->refresh_last_frame = 0; cpi->refresh_golden_frame = 0;
+#if CONFIG_EXT_REFS
+      cpi->refresh_bwd_ref_frame = 0;
+#endif  // CONFIG_EXT_REFS
       cpi->refresh_alt_ref_frame = 1;
       break;
+#if CONFIG_EXT_REFS
+    case BRF_UPDATE:
+      cpi->refresh_last_frame = 0;
+      cpi->refresh_golden_frame = 0;
+      cpi->refresh_bwd_ref_frame = 1;
+      cpi->refresh_alt_ref_frame = 0;
+      cpi->rc.is_bwd_ref_frame = 1;
+      break;
+    case LAST_BIPRED_UPDATE:
+      cpi->refresh_last_frame = 0;
+      cpi->refresh_golden_frame = 0;
+      cpi->refresh_bwd_ref_frame = 0;
+      cpi->refresh_alt_ref_frame = 0;
+      cpi->rc.is_last_bipred_frame = 1;
+      break;
+    case BIPRED_UPDATE:
+      cpi->refresh_last_frame = 1;
+      cpi->refresh_golden_frame = 0;
+      cpi->refresh_bwd_ref_frame = 0;
+      cpi->refresh_alt_ref_frame = 0;
+      cpi->rc.is_bipred_frame = 1;
+      break;
+#endif  // CONFIG_EXT_REFS
     default: assert(0); break;
   }
 }

av1/encoder/firstpass.h

@@ -40,6 +40,14 @@ typedef struct {
 } FIRSTPASS_MB_STATS;
 #endif
 
+#if CONFIG_EXT_REFS
+// Length of the bi-predictive frame group (BFG)
+// NOTE: Currently each BFG contains one backward ref (BWF) frame plus a certain
+//       number of bi-predictive frames.
+#define BFG_INTERVAL 2
+#define MIN_BFG_INTERVAL 2
+#endif  // CONFIG_EXT_REFS
+
 #define VLOW_MOTION_THRESHOLD 950
 
 typedef struct {
@@ -73,7 +81,14 @@ typedef enum {
   GF_UPDATE = 2,
   ARF_UPDATE = 3,
   OVERLAY_UPDATE = 4,
+#if CONFIG_EXT_REFS
+  BRF_UPDATE = 5,          // Backward Reference Frame
+  LAST_BIPRED_UPDATE = 6,  // Last Bi-Predictive Frame
+  BIPRED_UPDATE = 7,       // Bi-Predictive Frame, but not the last one
+  FRAME_UPDATE_TYPES = 8
+#else
   FRAME_UPDATE_TYPES = 5
+#endif  // CONFIG_EXT_REFS
 } FRAME_UPDATE_TYPE;
 
 #define FC_ANIMATION_THRESH 0.15
@@ -90,6 +105,10 @@ typedef struct {
   unsigned char arf_src_offset[(MAX_LAG_BUFFERS * 2) + 1];
   unsigned char arf_update_idx[(MAX_LAG_BUFFERS * 2) + 1];
   unsigned char arf_ref_idx[(MAX_LAG_BUFFERS * 2) + 1];
+#if CONFIG_EXT_REFS
+  unsigned char brf_src_offset[(MAX_LAG_BUFFERS * 2) + 1];
+  unsigned char brf_pred_enabled[(MAX_LAG_BUFFERS * 2) + 1];
+#endif  // CONFIG_EXT_REFS
   int bit_allocation[(MAX_LAG_BUFFERS * 2) + 1];
 } GF_GROUP;
 

av1/encoder/ratectrl.c

@@ -226,12 +226,17 @@ static void update_buffer_level(AV1_COMP *cpi, int encoded_frame_size) {
   const AV1_COMMON *const cm = &cpi->common;
   RATE_CONTROL *const rc = &cpi->rc;
 
-  // Non-viewable frames are a special case and are treated as pure overhead.
-  if (!cm->show_frame) {
+// Non-viewable frames are a special case and are treated as pure overhead.
+#if CONFIG_EXT_REFS
+  // TODO(zoeliu): To further explore whether we should treat BWDREF_FRAME
+  //               differently, since it is a no-show frame.
+  if (!cm->show_frame && !rc->is_bwd_ref_frame)
+#else
+  if (!cm->show_frame)
+#endif  // CONFIG_EXT_REFS
     rc->bits_off_target -= encoded_frame_size;
-  } else {
+  else
     rc->bits_off_target += rc->avg_frame_bandwidth - encoded_frame_size;
-  }
 
   // Clip the buffer level to the maximum specified buffer size.
   rc->bits_off_target = AOMMIN(rc->bits_off_target, rc->maximum_buffer_size);
@@ -908,13 +913,31 @@ static int rc_pick_q_and_bounds_one_pass_vbr(const AV1_COMP *cpi,
 int av1_frame_type_qdelta(const AV1_COMP *cpi, int rf_level, int q) {
   static const double rate_factor_deltas[RATE_FACTOR_LEVELS] = {
     1.00,  // INTER_NORMAL
+#if CONFIG_EXT_REFS
+    0.80,  // INTER_LOW
+    1.25,  // INTER_HIGH
+#else
     1.00,  // INTER_HIGH
+#endif     // CONFIG_EXT_REFS
     1.50,  // GF_ARF_LOW
     1.75,  // GF_ARF_STD
     2.00,  // KF_STD
   };
   static const FRAME_TYPE frame_type[RATE_FACTOR_LEVELS] = {
-    INTER_FRAME, INTER_FRAME, INTER_FRAME, INTER_FRAME, KEY_FRAME
+#if CONFIG_EXT_REFS
+    INTER_FRAME,
+    INTER_FRAME,
+    INTER_FRAME,
+    INTER_FRAME,
+    INTER_FRAME,
+    KEY_FRAME
+#else
+    INTER_FRAME,
+    INTER_FRAME,
+    INTER_FRAME,
+    INTER_FRAME,
+    KEY_FRAME
+#endif  // CONFIG_EXT_REFS
   };
   const AV1_COMMON *const cm = &cpi->common;
   int qdelta =
@@ -1262,7 +1285,12 @@ void av1_rc_postencode_update(AV1_COMP *cpi, uint64_t bytes_used) {
 
   // Actual bits spent
   rc->total_actual_bits += rc->projected_frame_size;
+#if CONFIG_EXT_REFS
+  rc->total_target_bits +=
+      (cm->show_frame || rc->is_bwd_ref_frame) ? rc->avg_frame_bandwidth : 0;
+#else
   rc->total_target_bits += cm->show_frame ? rc->avg_frame_bandwidth : 0;
+#endif  // CONFIG_EXT_REFS
 
   rc->total_target_vs_actual = rc->total_actual_bits - rc->total_target_bits;
 
@@ -1275,7 +1303,11 @@ void av1_rc_postencode_update(AV1_COMP *cpi, uint64_t bytes_used) {
     update_golden_frame_stats(cpi);
 
   if (cm->frame_type == KEY_FRAME) rc->frames_since_key = 0;
+#if CONFIG_EXT_REFS
+  if (cm->show_frame || rc->is_bwd_ref_frame) {
+#else
   if (cm->show_frame) {
+#endif  // CONFIG_EXT_REFS
     rc->frames_since_key++;
     rc->frames_to_key--;
   }

av1/encoder/ratectrl.h

@@ -28,6 +28,17 @@ extern "C" {
 #define MAX_GF_INTERVAL 16
 #define FIXED_GF_INTERVAL 8  // Used in some testing modes only
 
+#if CONFIG_EXT_REFS
+typedef enum {
+  INTER_NORMAL = 0,
+  INTER_LOW = 1,
+  INTER_HIGH = 2,
+  GF_ARF_LOW = 3,
+  GF_ARF_STD = 5,
+  KF_STD = 6,
+  RATE_FACTOR_LEVELS = 7
+} RATE_FACTOR_LEVEL;
+#else
 typedef enum {
   INTER_NORMAL = 0,
   INTER_HIGH = 1,
@@ -36,6 +47,7 @@ typedef enum {
   KF_STD = 4,
   RATE_FACTOR_LEVELS = 5
 } RATE_FACTOR_LEVEL;
+#endif  // CONFIG_EXT_REFS
 
 // Internal frame scaling level.
 typedef enum {
@@ -90,6 +102,17 @@ typedef struct {
   int source_alt_ref_active;
   int is_src_frame_alt_ref;
 
+#if CONFIG_EXT_REFS
+  // Length of the bi-predictive frame group interval
+  int bipred_group_interval;
+
+  // NOTE: Different types of frames may have different bits allocated
+  //       accordingly, aiming to achieve the overall optimal RD performance.
+  int is_bwd_ref_frame;
+  int is_last_bipred_frame;
+  int is_bipred_frame;
+#endif  // CONFIG_EXT_REFS
+
   int avg_frame_bandwidth;  // Average frame size target for clip
   int min_frame_bandwidth;  // Minimum allocation used for any frame
   int max_frame_bandwidth;  // Maximum burst rate allowed for a frame.