diff --git a/av1/common/odintrin.h b/av1/common/odintrin.h index 830e0c00e..a5e36cf64 100644 --- a/av1/common/odintrin.h +++ b/av1/common/odintrin.h @@ -11,10 +11,10 @@ #ifndef AV1_COMMON_ODINTRIN_H_ #define AV1_COMMON_ODINTRIN_H_ -#include "av1/common/enums.h" #include "aom/aom_integer.h" #include "aom_dsp/aom_dsp_common.h" #include "aom_ports/bitops.h" +#include "av1/common/enums.h" #ifdef __cplusplus extern "C" { diff --git a/av1/encoder/encoder.h b/av1/encoder/encoder.h index c2f8631a3..c7c1ff7a8 100644 --- a/av1/encoder/encoder.h +++ b/av1/encoder/encoder.h @@ -324,6 +324,9 @@ typedef struct AV1_COMP { int refresh_last_frame; int refresh_golden_frame; +#if CONFIG_EXT_REFS + int refresh_bwd_ref_frame; +#endif // CONFIG_EXT_REFS int refresh_alt_ref_frame; int ext_refresh_frame_flags_pending; diff --git a/av1/encoder/firstpass.c b/av1/encoder/firstpass.c index 03441fa96..765746b92 100644 --- a/av1/encoder/firstpass.c +++ b/av1/encoder/firstpass.c @@ -23,6 +23,7 @@ #include "aom_scale/aom_scale.h" #include "aom_scale/yv12config.h" +#include "aom_dsp/variance.h" #include "av1/common/entropymv.h" #include "av1/common/quant_common.h" #include "av1/common/reconinter.h" // av1_setup_dst_planes() @@ -37,7 +38,6 @@ #include "av1/encoder/mcomp.h" #include "av1/encoder/quantize.h" #include "av1/encoder/rd.h" -#include "aom_dsp/variance.h" #define OUTPUT_FPF 0 #define ARF_STATS_OUTPUT 0 @@ -1536,7 +1536,7 @@ static void allocate_gf_group_bits(AV1_COMP *cpi, int64_t gf_group_bits, GF_GROUP *const gf_group = &twopass->gf_group; FIRSTPASS_STATS frame_stats; int i; - int frame_index = 1; + int frame_index = 0; int target_frame_size; int key_frame; const int max_bits = frame_max_bits(&cpi->rc, &cpi->oxcf); @@ -1546,7 +1546,21 @@ static void allocate_gf_group_bits(AV1_COMP *cpi, int64_t gf_group_bits, int mid_boost_bits = 0; int mid_frame_idx; unsigned char arf_buffer_indices[MAX_ACTIVE_ARFS]; - int alt_frame_index = frame_index; + +#if CONFIG_EXT_REFS + // The use of bi-predictive frames are only enabled when following 3 + // conditions are met: + // (1) Alt-ref is enabled; + // (2) The bi-predictive group interval is at least 2; and + // (3) The bi-predictive group interval is strictly smaller than the + // golden group interval. + const int is_bipred_enabled = + rc->source_alt_ref_pending && rc->bipred_group_interval && + rc->bipred_group_interval <= + (rc->baseline_gf_interval - rc->source_alt_ref_pending); + int bipred_group_end = 0; + int bipred_frame_index = 0; +#endif // CONFIG_EXT_REFS key_frame = cpi->common.frame_type == KEY_FRAME; @@ -1556,40 +1570,54 @@ static void allocate_gf_group_bits(AV1_COMP *cpi, int64_t gf_group_bits, // is also the golden frame. if (!key_frame) { if (rc->source_alt_ref_active) { - gf_group->update_type[0] = OVERLAY_UPDATE; - gf_group->rf_level[0] = INTER_NORMAL; - gf_group->bit_allocation[0] = 0; - gf_group->arf_update_idx[0] = arf_buffer_indices[0]; - gf_group->arf_ref_idx[0] = arf_buffer_indices[0]; + gf_group->update_type[frame_index] = OVERLAY_UPDATE; + gf_group->rf_level[frame_index] = INTER_NORMAL; + gf_group->bit_allocation[frame_index] = 0; } else { - gf_group->update_type[0] = GF_UPDATE; - gf_group->rf_level[0] = GF_ARF_STD; - gf_group->bit_allocation[0] = gf_arf_bits; - gf_group->arf_update_idx[0] = arf_buffer_indices[0]; - gf_group->arf_ref_idx[0] = arf_buffer_indices[0]; + gf_group->update_type[frame_index] = GF_UPDATE; + gf_group->rf_level[frame_index] = GF_ARF_STD; + gf_group->bit_allocation[frame_index] = gf_arf_bits; } + gf_group->arf_update_idx[frame_index] = arf_buffer_indices[0]; + gf_group->arf_ref_idx[frame_index] = arf_buffer_indices[0]; // Step over the golden frame / overlay frame if (EOF == input_stats(twopass, &frame_stats)) return; } +#if CONFIG_EXT_REFS + gf_group->brf_src_offset[frame_index] = 0; + gf_group->brf_pred_enabled[frame_index] = 0; +#endif // CONFIG_EXT_REFS + // Deduct the boost bits for arf (or gf if it is not a key frame) // from the group total. if (rc->source_alt_ref_pending || !key_frame) total_group_bits -= gf_arf_bits; + frame_index++; +#if CONFIG_EXT_REFS + bipred_frame_index++; +#endif // CONFIG_EXT_REFS + // Store the bits to spend on the ARF if there is one. if (rc->source_alt_ref_pending) { - gf_group->update_type[alt_frame_index] = ARF_UPDATE; - gf_group->rf_level[alt_frame_index] = GF_ARF_STD; - gf_group->bit_allocation[alt_frame_index] = gf_arf_bits; + gf_group->update_type[frame_index] = ARF_UPDATE; + gf_group->rf_level[frame_index] = GF_ARF_STD; + gf_group->bit_allocation[frame_index] = gf_arf_bits; - gf_group->arf_src_offset[alt_frame_index] = + gf_group->arf_src_offset[frame_index] = (unsigned char)(rc->baseline_gf_interval - 1); - gf_group->arf_update_idx[alt_frame_index] = arf_buffer_indices[0]; - gf_group->arf_ref_idx[alt_frame_index] = + gf_group->arf_update_idx[frame_index] = arf_buffer_indices[0]; + gf_group->arf_ref_idx[frame_index] = arf_buffer_indices[cpi->multi_arf_last_grp_enabled && rc->source_alt_ref_active]; +#if CONFIG_EXT_REFS + gf_group->brf_src_offset[frame_index] = 0; + gf_group->brf_pred_enabled[frame_index] = 0; +// NOTE: "bipred_frame_index" stays unchanged for ARF_UPDATE frames. +#endif // CONFIG_EXT_REFS + ++frame_index; if (cpi->multi_arf_enabled) { @@ -1633,10 +1661,73 @@ static void allocate_gf_group_bits(AV1_COMP *cpi, int64_t gf_group_bits, target_frame_size = clamp(target_frame_size, 0, AOMMIN(max_bits, (int)total_group_bits)); - gf_group->update_type[frame_index] = LF_UPDATE; - gf_group->rf_level[frame_index] = INTER_NORMAL; +#if CONFIG_EXT_REFS + // NOTE: Bi-predictive frames are only enabled when the length of the + // bi-predictive frame group interval is strictly smaller than that + // of the golden frame group interval. + // TODO(zoeliu): Currently bi-prediction is only enabled when alt-ref is on. + if (is_bipred_enabled && !bipred_group_end) { + const int cur_brf_src_offset = rc->bipred_group_interval - 1; + + // --- BRF_UPDATE --- + if (bipred_frame_index == 1) { + gf_group->update_type[frame_index] = BRF_UPDATE; + gf_group->brf_src_offset[frame_index] = cur_brf_src_offset; + gf_group->brf_pred_enabled[frame_index] = 1; + // --- LAST_BIPRED_UPDATE --- + } else if (bipred_frame_index == rc->bipred_group_interval) { + gf_group->update_type[frame_index] = LAST_BIPRED_UPDATE; + gf_group->brf_src_offset[frame_index] = 0; + gf_group->brf_pred_enabled[frame_index] = 1; + // Reset the bi-predictive frame index. + bipred_frame_index = 0; + // --- BIPRED_UPDATE --- + } else { + gf_group->update_type[frame_index] = BIPRED_UPDATE; + gf_group->brf_src_offset[frame_index] = 0; + gf_group->brf_pred_enabled[frame_index] = 1; + } + + bipred_frame_index++; + // Check whether the next bi-predictive frame group would entirely be + // included within the current golden frame group. + if (bipred_frame_index == 1 && + (i + 1 + cur_brf_src_offset) >= + (rc->baseline_gf_interval - rc->source_alt_ref_pending)) + bipred_group_end = 1; + } else { +#endif // CONFIG_EXT_REFS + gf_group->update_type[frame_index] = LF_UPDATE; +#if CONFIG_EXT_REFS + gf_group->brf_src_offset[frame_index] = 0; + gf_group->brf_pred_enabled[frame_index] = 0; + } +#endif // CONFIG_EXT_REFS + +#if CONFIG_EXT_REFS + if (gf_group->update_type[frame_index] == BRF_UPDATE) { + // Boost up the allocated bits on BWDREF_FRAME + gf_group->rf_level[frame_index] = INTER_HIGH; + gf_group->bit_allocation[frame_index] = + target_frame_size + (target_frame_size >> 2); + } else if (gf_group->update_type[frame_index] == LAST_BIPRED_UPDATE) { + // Press down the allocated bits on LAST_BIPRED_UPDATE frames + gf_group->rf_level[frame_index] = INTER_LOW; + gf_group->bit_allocation[frame_index] = + target_frame_size - (target_frame_size >> 1); + } else if (gf_group->update_type[frame_index] == BIPRED_UPDATE) { + // TODO(zoeliu): To investigate whether the allocated bits on + // BIPRED_UPDATE frames need to be further adjusted. + gf_group->rf_level[frame_index] = INTER_NORMAL; + gf_group->bit_allocation[frame_index] = target_frame_size; + } else { +#endif // CONFIG_EXT_REFS + gf_group->rf_level[frame_index] = INTER_NORMAL; + gf_group->bit_allocation[frame_index] = target_frame_size; +#if CONFIG_EXT_REFS + } +#endif // CONFIG_EXT_REFS - gf_group->bit_allocation[frame_index] = target_frame_size; ++frame_index; } @@ -1662,6 +1753,10 @@ static void allocate_gf_group_bits(AV1_COMP *cpi, int64_t gf_group_bits, gf_group->update_type[frame_index] = GF_UPDATE; gf_group->rf_level[frame_index] = GF_ARF_STD; } +#if CONFIG_EXT_REFS + gf_group->brf_src_offset[frame_index] = 0; + gf_group->brf_pred_enabled[frame_index] = 0; +#endif // CONFIG_EXT_REFS // Note whether multi-arf was enabled this group for next time. cpi->multi_arf_last_grp_enabled = cpi->multi_arf_enabled; @@ -1871,6 +1966,13 @@ static void define_gf_group(AV1_COMP *cpi, FIRSTPASS_STATS *this_frame) { rc->frames_till_gf_update_due = rc->baseline_gf_interval; +#if CONFIG_EXT_REFS + rc->bipred_group_interval = BFG_INTERVAL; + // The minimum bi-predictive frame group interval is 2. + if (rc->bipred_group_interval < MIN_BFG_INTERVAL) + rc->bipred_group_interval = 0; +#endif // CONFIG_EXT_REFS + // Reset the file position. reset_fpf_position(twopass, start_pos); @@ -2295,33 +2397,69 @@ static void configure_buffer_updates(AV1_COMP *cpi) { TWO_PASS *const twopass = &cpi->twopass; cpi->rc.is_src_frame_alt_ref = 0; +#if CONFIG_EXT_REFS + cpi->rc.is_bwd_ref_frame = 0; + cpi->rc.is_last_bipred_frame = 0; + cpi->rc.is_bipred_frame = 0; +#endif // CONFIG_EXT_REFS + switch (twopass->gf_group.update_type[twopass->gf_group.index]) { - case KF_UPDATE: - cpi->refresh_last_frame = 1; - cpi->refresh_golden_frame = 1; + case KF_UPDATE: cpi->refresh_last_frame = 1; cpi->refresh_golden_frame = 1; +#if CONFIG_EXT_REFS + cpi->refresh_bwd_ref_frame = 1; +#endif // CONFIG_EXT_REFS cpi->refresh_alt_ref_frame = 1; break; - case LF_UPDATE: - cpi->refresh_last_frame = 1; - cpi->refresh_golden_frame = 0; + case LF_UPDATE: cpi->refresh_last_frame = 1; cpi->refresh_golden_frame = 0; +#if CONFIG_EXT_REFS + cpi->refresh_bwd_ref_frame = 0; +#endif // CONFIG_EXT_REFS cpi->refresh_alt_ref_frame = 0; break; - case GF_UPDATE: - cpi->refresh_last_frame = 1; - cpi->refresh_golden_frame = 1; + case GF_UPDATE: cpi->refresh_last_frame = 1; cpi->refresh_golden_frame = 1; +#if CONFIG_EXT_REFS + cpi->refresh_bwd_ref_frame = 0; +#endif // CONFIG_EXT_REFS cpi->refresh_alt_ref_frame = 0; break; case OVERLAY_UPDATE: cpi->refresh_last_frame = 0; cpi->refresh_golden_frame = 1; +#if CONFIG_EXT_REFS + cpi->refresh_bwd_ref_frame = 0; +#endif // CONFIG_EXT_REFS cpi->refresh_alt_ref_frame = 0; cpi->rc.is_src_frame_alt_ref = 1; break; - case ARF_UPDATE: - cpi->refresh_last_frame = 0; - cpi->refresh_golden_frame = 0; + case ARF_UPDATE: cpi->refresh_last_frame = 0; cpi->refresh_golden_frame = 0; +#if CONFIG_EXT_REFS + cpi->refresh_bwd_ref_frame = 0; +#endif // CONFIG_EXT_REFS cpi->refresh_alt_ref_frame = 1; break; +#if CONFIG_EXT_REFS + case BRF_UPDATE: + cpi->refresh_last_frame = 0; + cpi->refresh_golden_frame = 0; + cpi->refresh_bwd_ref_frame = 1; + cpi->refresh_alt_ref_frame = 0; + cpi->rc.is_bwd_ref_frame = 1; + break; + case LAST_BIPRED_UPDATE: + cpi->refresh_last_frame = 0; + cpi->refresh_golden_frame = 0; + cpi->refresh_bwd_ref_frame = 0; + cpi->refresh_alt_ref_frame = 0; + cpi->rc.is_last_bipred_frame = 1; + break; + case BIPRED_UPDATE: + cpi->refresh_last_frame = 1; + cpi->refresh_golden_frame = 0; + cpi->refresh_bwd_ref_frame = 0; + cpi->refresh_alt_ref_frame = 0; + cpi->rc.is_bipred_frame = 1; + break; +#endif // CONFIG_EXT_REFS default: assert(0); break; } } diff --git a/av1/encoder/firstpass.h b/av1/encoder/firstpass.h index d9366c024..d2d929d31 100644 --- a/av1/encoder/firstpass.h +++ b/av1/encoder/firstpass.h @@ -40,6 +40,14 @@ typedef struct { } FIRSTPASS_MB_STATS; #endif +#if CONFIG_EXT_REFS +// Length of the bi-predictive frame group (BFG) +// NOTE: Currently each BFG contains one backward ref (BWF) frame plus a certain +// number of bi-predictive frames. +#define BFG_INTERVAL 2 +#define MIN_BFG_INTERVAL 2 +#endif // CONFIG_EXT_REFS + #define VLOW_MOTION_THRESHOLD 950 typedef struct { @@ -73,7 +81,14 @@ typedef enum { GF_UPDATE = 2, ARF_UPDATE = 3, OVERLAY_UPDATE = 4, +#if CONFIG_EXT_REFS + BRF_UPDATE = 5, // Backward Reference Frame + LAST_BIPRED_UPDATE = 6, // Last Bi-Predictive Frame + BIPRED_UPDATE = 7, // Bi-Predictive Frame, but not the last one + FRAME_UPDATE_TYPES = 8 +#else FRAME_UPDATE_TYPES = 5 +#endif // CONFIG_EXT_REFS } FRAME_UPDATE_TYPE; #define FC_ANIMATION_THRESH 0.15 @@ -90,6 +105,10 @@ typedef struct { unsigned char arf_src_offset[(MAX_LAG_BUFFERS * 2) + 1]; unsigned char arf_update_idx[(MAX_LAG_BUFFERS * 2) + 1]; unsigned char arf_ref_idx[(MAX_LAG_BUFFERS * 2) + 1]; +#if CONFIG_EXT_REFS + unsigned char brf_src_offset[(MAX_LAG_BUFFERS * 2) + 1]; + unsigned char brf_pred_enabled[(MAX_LAG_BUFFERS * 2) + 1]; +#endif // CONFIG_EXT_REFS int bit_allocation[(MAX_LAG_BUFFERS * 2) + 1]; } GF_GROUP; diff --git a/av1/encoder/ratectrl.c b/av1/encoder/ratectrl.c index 9ff1b5f8f..4477885b5 100644 --- a/av1/encoder/ratectrl.c +++ b/av1/encoder/ratectrl.c @@ -226,12 +226,17 @@ static void update_buffer_level(AV1_COMP *cpi, int encoded_frame_size) { const AV1_COMMON *const cm = &cpi->common; RATE_CONTROL *const rc = &cpi->rc; - // Non-viewable frames are a special case and are treated as pure overhead. - if (!cm->show_frame) { +// Non-viewable frames are a special case and are treated as pure overhead. +#if CONFIG_EXT_REFS + // TODO(zoeliu): To further explore whether we should treat BWDREF_FRAME + // differently, since it is a no-show frame. + if (!cm->show_frame && !rc->is_bwd_ref_frame) +#else + if (!cm->show_frame) +#endif // CONFIG_EXT_REFS rc->bits_off_target -= encoded_frame_size; - } else { + else rc->bits_off_target += rc->avg_frame_bandwidth - encoded_frame_size; - } // Clip the buffer level to the maximum specified buffer size. rc->bits_off_target = AOMMIN(rc->bits_off_target, rc->maximum_buffer_size); @@ -908,13 +913,31 @@ static int rc_pick_q_and_bounds_one_pass_vbr(const AV1_COMP *cpi, int av1_frame_type_qdelta(const AV1_COMP *cpi, int rf_level, int q) { static const double rate_factor_deltas[RATE_FACTOR_LEVELS] = { 1.00, // INTER_NORMAL +#if CONFIG_EXT_REFS + 0.80, // INTER_LOW + 1.25, // INTER_HIGH +#else 1.00, // INTER_HIGH +#endif // CONFIG_EXT_REFS 1.50, // GF_ARF_LOW 1.75, // GF_ARF_STD 2.00, // KF_STD }; static const FRAME_TYPE frame_type[RATE_FACTOR_LEVELS] = { - INTER_FRAME, INTER_FRAME, INTER_FRAME, INTER_FRAME, KEY_FRAME +#if CONFIG_EXT_REFS + INTER_FRAME, + INTER_FRAME, + INTER_FRAME, + INTER_FRAME, + INTER_FRAME, + KEY_FRAME +#else + INTER_FRAME, + INTER_FRAME, + INTER_FRAME, + INTER_FRAME, + KEY_FRAME +#endif // CONFIG_EXT_REFS }; const AV1_COMMON *const cm = &cpi->common; int qdelta = @@ -1262,7 +1285,12 @@ void av1_rc_postencode_update(AV1_COMP *cpi, uint64_t bytes_used) { // Actual bits spent rc->total_actual_bits += rc->projected_frame_size; +#if CONFIG_EXT_REFS + rc->total_target_bits += + (cm->show_frame || rc->is_bwd_ref_frame) ? rc->avg_frame_bandwidth : 0; +#else rc->total_target_bits += cm->show_frame ? rc->avg_frame_bandwidth : 0; +#endif // CONFIG_EXT_REFS rc->total_target_vs_actual = rc->total_actual_bits - rc->total_target_bits; @@ -1275,7 +1303,11 @@ void av1_rc_postencode_update(AV1_COMP *cpi, uint64_t bytes_used) { update_golden_frame_stats(cpi); if (cm->frame_type == KEY_FRAME) rc->frames_since_key = 0; +#if CONFIG_EXT_REFS + if (cm->show_frame || rc->is_bwd_ref_frame) { +#else if (cm->show_frame) { +#endif // CONFIG_EXT_REFS rc->frames_since_key++; rc->frames_to_key--; } diff --git a/av1/encoder/ratectrl.h b/av1/encoder/ratectrl.h index b5fa3381c..313c700d9 100644 --- a/av1/encoder/ratectrl.h +++ b/av1/encoder/ratectrl.h @@ -28,6 +28,17 @@ extern "C" { #define MAX_GF_INTERVAL 16 #define FIXED_GF_INTERVAL 8 // Used in some testing modes only +#if CONFIG_EXT_REFS +typedef enum { + INTER_NORMAL = 0, + INTER_LOW = 1, + INTER_HIGH = 2, + GF_ARF_LOW = 3, + GF_ARF_STD = 5, + KF_STD = 6, + RATE_FACTOR_LEVELS = 7 +} RATE_FACTOR_LEVEL; +#else typedef enum { INTER_NORMAL = 0, INTER_HIGH = 1, @@ -36,6 +47,7 @@ typedef enum { KF_STD = 4, RATE_FACTOR_LEVELS = 5 } RATE_FACTOR_LEVEL; +#endif // CONFIG_EXT_REFS // Internal frame scaling level. typedef enum { @@ -90,6 +102,17 @@ typedef struct { int source_alt_ref_active; int is_src_frame_alt_ref; +#if CONFIG_EXT_REFS + // Length of the bi-predictive frame group interval + int bipred_group_interval; + + // NOTE: Different types of frames may have different bits allocated + // accordingly, aiming to achieve the overall optimal RD performance. + int is_bwd_ref_frame; + int is_last_bipred_frame; + int is_bipred_frame; +#endif // CONFIG_EXT_REFS + int avg_frame_bandwidth; // Average frame size target for clip int min_frame_bandwidth; // Minimum allocation used for any frame int max_frame_bandwidth; // Maximum burst rate allowed for a frame.