Fork the entropy experiment

Split it into two experiments:
q_adapt_probs: multiple initial coeff prob tables based on q-index
subframe_prob_update: multiple backward prob updates within frame

Change-Id: I78041ebd4ba34afc9152f6861225f63c2e8eb686

av1/common/entropy.c

@@ -1079,7 +1079,7 @@ const aom_cdf_prob
 #endif  // CONFIG_NEW_TOKENSET
 
 /* clang-format off */
-#if CONFIG_ENTROPY
+#if CONFIG_Q_ADAPT_PROBS
 const av1_coeff_probs_model
 default_qctx_coef_probs[QCTX_BINS][TX_SIZES][PLANE_TYPES] = {
     {  // Q_Index 0
@@ -4597,7 +4597,7 @@ static const av1_coeff_probs_model default_coef_probs_64x64[PLANE_TYPES] = {
   }
 };
 #endif  // CONFIG_TX64X64
-#endif  // CONFIG_ENTROPY
+#endif  // CONFIG_Q_ADAPT_PROBS
 #if CONFIG_NEW_TOKENSET
 static const aom_prob av1_default_blockzero_probs[TX_SIZES][PLANE_TYPES]
                                            [REF_TYPES][BLOCKZ_CONTEXTS] = {
@@ -5490,7 +5490,7 @@ void av1_coef_pareto_cdfs(FRAME_CONTEXT *fc) {
 #endif
 
 void av1_default_coef_probs(AV1_COMMON *cm) {
-#if CONFIG_ENTROPY
+#if CONFIG_Q_ADAPT_PROBS
   const int index = AOMMIN(
       ROUND_POWER_OF_TWO(cm->base_qindex, 8 - QCTX_BIN_BITS), QCTX_BINS - 1);
   av1_copy(cm->fc->coef_probs, default_qctx_coef_probs[index]);
@@ -5505,7 +5505,7 @@ void av1_default_coef_probs(AV1_COMMON *cm) {
 #if CONFIG_TX64X64
   av1_copy(cm->fc->coef_probs[TX_64X64], default_coef_probs_64x64);
 #endif  // CONFIG_TX64X64
-#endif  // CONFIG_ENTROPY
+#endif  // CONFIG_Q_ADAPT_PROBS
 #if CONFIG_NEW_TOKENSET
   av1_copy(cm->fc->blockzero_probs, av1_default_blockzero_probs);
 #endif
@@ -5523,14 +5523,14 @@ static void adapt_coef_probs(AV1_COMMON *cm, TX_SIZE tx_size,
                              unsigned int update_factor) {
   const FRAME_CONTEXT *pre_fc = &cm->frame_contexts[cm->frame_context_idx];
   av1_coeff_probs_model *const probs = cm->fc->coef_probs[tx_size];
-#if CONFIG_ENTROPY
+#if CONFIG_SUBFRAME_PROB_UPDATE
   const av1_coeff_probs_model *const pre_probs =
       cm->partial_prob_update
           ? (const av1_coeff_probs_model *)cm->starting_coef_probs[tx_size]
           : pre_fc->coef_probs[tx_size];
 #else
   const av1_coeff_probs_model *const pre_probs = pre_fc->coef_probs[tx_size];
-#endif  // CONFIG_ENTROPY
+#endif  // CONFIG_SUBFRAME_PROB_UPDATE
   const av1_coeff_count_model *const counts =
       (const av1_coeff_count_model *)cm->counts.coef[tx_size];
   const unsigned int(*eob_counts)[REF_TYPES][COEF_BANDS][COEFF_CONTEXTS] =
@@ -5595,16 +5595,16 @@ void av1_adapt_coef_probs(AV1_COMMON *cm) {
     update_factor = COEF_MAX_UPDATE_FACTOR;
     count_sat = COEF_COUNT_SAT;
   }
-#if CONFIG_ENTROPY
+#if CONFIG_SUBFRAME_PROB_UPDATE
   if (cm->partial_prob_update == 1) update_factor = COEF_MAX_UPDATE_FACTOR;
-#endif  // CONFIG_ENTROPY
+#endif  // CONFIG_SUBFRAME_PROB_UPDATE
 
   for (tx_size = 0; tx_size < TX_SIZES; tx_size++)
     adapt_coef_probs(cm, tx_size, count_sat, update_factor);
 
-#if CONFIG_ENTROPY
+#if CONFIG_SUBFRAME_PROB_UPDATE
   if (cm->partial_prob_update == 0)
-#endif  // CONFIG_ENTROPY
+#endif  // CONFIG_SUBFRAME_PROB_UPDATE
   {
 #if CONFIG_ADAPT_SCAN
     for (tx_size = 0; tx_size < TX_SIZES_ALL; ++tx_size) {
@@ -5622,7 +5622,7 @@ void av1_adapt_coef_probs(AV1_COMMON *cm) {
   }
 }
 
-#if CONFIG_ENTROPY
+#if CONFIG_SUBFRAME_PROB_UPDATE
 void av1_partial_adapt_probs(AV1_COMMON *cm, int mi_row, int mi_col) {
   (void)mi_row;
   (void)mi_col;
@@ -5632,4 +5632,4 @@ void av1_partial_adapt_probs(AV1_COMMON *cm, int mi_row, int mi_col) {
     av1_adapt_coef_probs(cm);
   }
 }
-#endif  // CONFIG_ENTROPY
+#endif  // CONFIG_SUBFRAME_PROB_UPDATE

av1/common/entropy.h

@@ -26,11 +26,14 @@ extern "C" {
 #define DIFF_UPDATE_PROB 252
 #define GROUP_DIFF_UPDATE_PROB 252
 
-#if CONFIG_ENTROPY
-#define COEF_PROBS_BUFS 16
+#if CONFIG_Q_ADAPT_PROBS
 #define QCTX_BIN_BITS 2
 #define QCTX_BINS (1 << QCTX_BIN_BITS)
-#endif  // CONFIG_ENTROPY
+#endif  // CONFIG_Q_ADAPT_PROBS
+
+#if CONFIG_SUBFRAME_PROB_UPDATE
+#define COEF_PROBS_BUFS 16
+#endif  // CONFIG_SUBFRAME_PROB_UPDATE
 
 // Coefficient token alphabet
 #define ZERO_TOKEN 0        // 0     Extra Bits 0+0
@@ -172,9 +175,9 @@ typedef unsigned int av1_coeff_stats[REF_TYPES][COEF_BANDS][COEFF_CONTEXTS]
 struct AV1Common;
 void av1_default_coef_probs(struct AV1Common *cm);
 void av1_adapt_coef_probs(struct AV1Common *cm);
-#if CONFIG_ENTROPY
+#if CONFIG_SUBFRAME_PROB_UPDATE
 void av1_partial_adapt_probs(struct AV1Common *cm, int mi_row, int mi_col);
-#endif  // CONFIG_ENTROPY
+#endif  // CONFIG_SUBFRAME_PROB_UPDATE
 
 // This is the index in the scan order beyond which all coefficients for
 // 8x8 transform and above are in the top band.

av1/common/onyxc_int.h

@@ -345,7 +345,7 @@ typedef struct AV1Common {
   unsigned int frame_context_idx; /* Context to use/update */
   FRAME_COUNTS counts;
 
-#if CONFIG_ENTROPY
+#if CONFIG_SUBFRAME_PROB_UPDATE
   // The initial probabilities for a frame, before any subframe backward update,
   // and after forward update.
   av1_coeff_probs_model starting_coef_probs[TX_SIZES][PLANE_TYPES];
@@ -355,7 +355,7 @@ typedef struct AV1Common {
   uint8_t partial_prob_update;
   // Frame level flag to turn on/off subframe backward update
   uint8_t do_subframe_update;
-#endif  // CONFIG_ENTROPY
+#endif  // CONFIG_SUBFRAME_PROB_UPDATE
 
   unsigned int current_video_frame;
   BITSTREAM_PROFILE profile;

av1/decoder/decodeframe.c

@@ -3470,9 +3470,9 @@ static const uint8_t *decode_tiles(AV1Decoder *pbi, const uint8_t *data,
 #endif  // CONFIG_EXT_TILE
   int tile_row, tile_col;
 
-#if CONFIG_ENTROPY
+#if CONFIG_SUBFRAME_PROB_UPDATE
   cm->do_subframe_update = n_tiles == 1;
-#endif  // CONFIG_ENTROPY
+#endif  // CONFIG_SUBFRAME_PROB_UPDATE
 
   if (cm->lf.filter_level && !cm->skip_loop_filter &&
       pbi->lf_worker.data1 == NULL) {
@@ -3612,7 +3612,7 @@ static const uint8_t *decode_tiles(AV1Decoder *pbi, const uint8_t *data,
         if (pbi->mb.corrupted)
           aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME,
                              "Failed to decode tile data");
-#if CONFIG_ENTROPY
+#if CONFIG_SUBFRAME_PROB_UPDATE
         if (cm->do_subframe_update &&
             cm->refresh_frame_context == REFRESH_FRAME_CONTEXT_BACKWARD) {
           const int mi_rows_per_update =
@@ -3624,7 +3624,7 @@ static const uint8_t *decode_tiles(AV1Decoder *pbi, const uint8_t *data,
             ++cm->coef_probs_update_idx;
           }
         }
-#endif  // CONFIG_ENTROPY
+#endif  // CONFIG_SUBFRAME_PROB_UPDATE
       }
     }
 
@@ -4372,7 +4372,7 @@ static size_t read_uncompressed_header(AV1Decoder *pbi,
   xd->bd = (int)cm->bit_depth;
 #endif
 
-#if CONFIG_ENTROPY
+#if CONFIG_Q_ADAPT_PROBS
   av1_default_coef_probs(cm);
   if (cm->frame_type == KEY_FRAME || cm->error_resilient_mode ||
       cm->reset_frame_context == RESET_FRAME_CONTEXT_ALL) {
@@ -4380,7 +4380,7 @@ static size_t read_uncompressed_header(AV1Decoder *pbi,
   } else if (cm->reset_frame_context == RESET_FRAME_CONTEXT_CURRENT) {
     cm->frame_contexts[cm->frame_context_idx] = *cm->fc;
   }
-#endif  // CONFIG_ENTROPY
+#endif  // CONFIG_Q_ADAPT_PROBS
 
   setup_segmentation(cm, rb);
 
@@ -4959,10 +4959,10 @@ void av1_decode_frame(AV1Decoder *pbi, const uint8_t *data,
     av1_frameworker_unlock_stats(worker);
   }
 
-#if CONFIG_ENTROPY
+#if CONFIG_SUBFRAME_PROB_UPDATE
   av1_copy(cm->starting_coef_probs, cm->fc->coef_probs);
   cm->coef_probs_update_idx = 0;
-#endif  // CONFIG_ENTROPY
+#endif  // CONFIG_SUBFRAME_PROB_UPDATE
 
   if (pbi->max_threads > 1
 #if CONFIG_EXT_TILE
@@ -5022,9 +5022,9 @@ void av1_decode_frame(AV1Decoder *pbi, const uint8_t *data,
 
   if (!xd->corrupted) {
     if (cm->refresh_frame_context == REFRESH_FRAME_CONTEXT_BACKWARD) {
-#if CONFIG_ENTROPY
+#if CONFIG_SUBFRAME_PROB_UPDATE
       cm->partial_prob_update = 0;
-#endif  // CONFIG_ENTROPY
+#endif  // CONFIG_SUBFRAME_PROB_UPDATE
       av1_adapt_coef_probs(cm);
       av1_adapt_intra_frame_probs(cm);
 

av1/encoder/bitstream.c

@@ -3090,7 +3090,7 @@ static void update_coef_probs_common(aom_writer *const bc, AV1_COMP *cpi,
   }
 }
 #endif
-#if CONFIG_ENTROPY
+#if CONFIG_SUBFRAME_PROB_UPDATE
 // Calculate the token counts between subsequent subframe updates.
 static void get_coef_counts_diff(
     AV1_COMP *cpi, int index,
@@ -3296,14 +3296,14 @@ static void update_coef_probs_subframe(
     default: assert(0);
   }
 }
-#endif  // CONFIG_ENTROPY
+#endif  // CONFIG_SUBFRAME_PROB_UPDATE
 
 #if !(CONFIG_EC_ADAPT && CONFIG_NEW_TOKENSET)
 static void update_coef_probs(AV1_COMP *cpi, aom_writer *w) {
   const TX_MODE tx_mode = cpi->common.tx_mode;
   const TX_SIZE max_tx_size = tx_mode_to_biggest_tx_size[tx_mode];
   TX_SIZE tx_size;
-#if CONFIG_ENTROPY
+#if CONFIG_SUBFRAME_PROB_UPDATE
   AV1_COMMON *cm = &cpi->common;
   SUBFRAME_STATS *subframe_stats = &cpi->subframe_stats;
   int i;
@@ -3318,7 +3318,7 @@ static void update_coef_probs(AV1_COMP *cpi, aom_writer *w) {
                            cpi->wholeframe_stats.eob_counts_buf[i]);
     }
   }
-#endif  // CONFIG_ENTROPY
+#endif  // CONFIG_SUBFRAME_PROB_UPDATE
 
   for (tx_size = 0; tx_size <= max_tx_size; ++tx_size) {
     av1_coeff_stats frame_branch_ct[PLANE_TYPES];
@@ -3327,7 +3327,7 @@ static void update_coef_probs(AV1_COMP *cpi, aom_writer *w) {
         (tx_size >= TX_16X16 && cpi->sf.tx_size_search_method == USE_TX_8X8)) {
       aom_write_bit(w, 0);
     } else {
-#if CONFIG_ENTROPY
+#if CONFIG_SUBFRAME_PROB_UPDATE
       if (cm->do_subframe_update &&
           cm->refresh_frame_context == REFRESH_FRAME_CONTEXT_BACKWARD) {
         unsigned int this_eob_counts_copy[PLANE_TYPES][REF_TYPES][COEF_BANDS]
@@ -3351,18 +3351,18 @@ static void update_coef_probs(AV1_COMP *cpi, aom_writer *w) {
         update_coef_probs_subframe(w, cpi, tx_size, cpi->branch_ct_buf,
                                    frame_coef_probs);
       } else {
-#endif  // CONFIG_ENTROPY
+#endif  // CONFIG_SUBFRAME_PROB_UPDATE
         build_tree_distribution(cpi, tx_size, frame_branch_ct,
                                 frame_coef_probs);
         update_coef_probs_common(w, cpi, tx_size, frame_branch_ct,
                                  frame_coef_probs);
-#if CONFIG_ENTROPY
+#if CONFIG_SUBFRAME_PROB_UPDATE
       }
-#endif  // CONFIG_ENTROPY
+#endif  // CONFIG_SUBFRAME_PROB_UPDATE
     }
   }
 
-#if CONFIG_ENTROPY
+#if CONFIG_SUBFRAME_PROB_UPDATE
   av1_copy(cm->starting_coef_probs, cm->fc->coef_probs);
   av1_copy(subframe_stats->coef_probs_buf[0], cm->fc->coef_probs);
   if (cm->do_subframe_update &&
@@ -3381,7 +3381,7 @@ static void update_coef_probs(AV1_COMP *cpi, aom_writer *w) {
     av1_copy(cm->fc->coef_probs, subframe_stats->coef_probs_buf[0]);
     av1_copy(cm->counts.eob_branch, eob_counts_copy);
   }
-#endif  // CONFIG_ENTROPY
+#endif  // CONFIG_SUBFRAME_PROB_UPDATE
 }
 #endif
 #endif  // !CONFIG_EC_ADAPT

av1/encoder/encodeframe.c

@@ -4673,7 +4673,7 @@ static void encode_rd_sb_row(AV1_COMP *cpi, ThreadData *td,
                         INT64_MAX, pc_root);
     }
   }
-#if CONFIG_ENTROPY
+#if CONFIG_SUBFRAME_PROB_UPDATE
   if (cm->do_subframe_update &&
       cm->refresh_frame_context == REFRESH_FRAME_CONTEXT_BACKWARD) {
     const int mi_rows_per_update =
@@ -4698,7 +4698,7 @@ static void encode_rd_sb_row(AV1_COMP *cpi, ThreadData *td,
       av1_fill_token_costs(x->token_costs, cm->fc->coef_probs);
     }
   }
-#endif  // CONFIG_ENTROPY
+#endif  // CONFIG_SUBFRAME_PROB_UPDATE
 }
 
 static void init_encode_frame_mb_context(AV1_COMP *cpi) {

av1/encoder/encoder.c

@@ -4038,13 +4038,13 @@ static void encode_without_recode_loop(AV1_COMP *cpi) {
 
   setup_frame(cpi);
 
-#if CONFIG_ENTROPY
+#if CONFIG_SUBFRAME_PROB_UPDATE
   cm->do_subframe_update = cm->tile_cols == 1 && cm->tile_rows == 1;
   av1_copy(cm->starting_coef_probs, cm->fc->coef_probs);
   av1_copy(cpi->subframe_stats.enc_starting_coef_probs, cm->fc->coef_probs);
   cm->coef_probs_update_idx = 0;
   av1_copy(cpi->subframe_stats.coef_probs_buf[0], cm->fc->coef_probs);
-#endif  // CONFIG_ENTROPY
+#endif  // CONFIG_SUBFRAME_PROB_UPDATE
 
   suppress_active_map(cpi);
   // Variance adaptive and in frame q adjustment experiments are mutually
@@ -4152,7 +4152,7 @@ static void encode_with_recode_loop(AV1_COMP *cpi, size_t *size,
 
     if (loop_count == 0) setup_frame(cpi);
 
-#if CONFIG_ENTROPY
+#if CONFIG_Q_ADAPT_PROBS
     // Base q-index may have changed, so we need to assign proper default coef
     // probs before every iteration.
     if (frame_is_intra_only(cm) || cm->error_resilient_mode) {
@@ -4165,7 +4165,9 @@ static void encode_with_recode_loop(AV1_COMP *cpi, size_t *size,
         cm->frame_contexts[cm->frame_context_idx] = *cm->fc;
       }
     }
+#endif  // CONFIG_Q_ADAPT_PROBS
 
+#if CONFIG_SUBFRAME_PROB_UPDATE
     cm->do_subframe_update = cm->tile_cols == 1 && cm->tile_rows == 1;
     if (loop_count == 0 || frame_is_intra_only(cm) ||
         cm->error_resilient_mode) {
@@ -4183,7 +4185,7 @@ static void encode_with_recode_loop(AV1_COMP *cpi, size_t *size,
     }
     cm->coef_probs_update_idx = 0;
     av1_copy(cpi->subframe_stats.coef_probs_buf[0], cm->fc->coef_probs);
-#endif  // CONFIG_ENTROPY
+#endif  // CONFIG_SUBFRAME_PROB_UPDATE
 
     // Variance adaptive and in frame q adjustment experiments are mutually
     // exclusive.
@@ -4916,9 +4918,9 @@ static void encode_frame_to_data_rate(AV1_COMP *cpi, size_t *size,
   av1_accumulate_frame_counts(&aggregate_fc, &cm->counts);
 #endif  // CONFIG_ENTROPY_STATS
   if (cm->refresh_frame_context == REFRESH_FRAME_CONTEXT_BACKWARD) {
-#if CONFIG_ENTROPY
+#if CONFIG_SUBFRAME_PROB_UPDATE
     cm->partial_prob_update = 0;
-#endif  // CONFIG_ENTROPY
+#endif  // CONFIG_SUBFRAME_PROB_UPDATE
     av1_adapt_coef_probs(cm);
     av1_adapt_intra_frame_probs(cm);
   }

av1/encoder/encoder.h

@@ -338,7 +338,7 @@ typedef struct {
   YV12_BUFFER_CONFIG buf;
 } EncRefCntBuffer;
 
-#if CONFIG_ENTROPY
+#if CONFIG_SUBFRAME_PROB_UPDATE
 typedef struct SUBFRAME_STATS {
   av1_coeff_probs_model coef_probs_buf[COEF_PROBS_BUFS][TX_SIZES][PLANE_TYPES];
   av1_coeff_count coef_counts_buf[COEF_PROBS_BUFS][TX_SIZES][PLANE_TYPES];
@@ -346,7 +346,7 @@ typedef struct SUBFRAME_STATS {
                              [COEF_BANDS][COEFF_CONTEXTS];
   av1_coeff_probs_model enc_starting_coef_probs[TX_SIZES][PLANE_TYPES];
 } SUBFRAME_STATS;
-#endif  // CONFIG_ENTROPY
+#endif  // CONFIG_SUBFRAME_PROB_UPDATE
 
 typedef struct TileBufferEnc {
   uint8_t *data;
@@ -642,12 +642,12 @@ typedef struct AV1_COMP {
   AVxWorker *workers;
   struct EncWorkerData *tile_thr_data;
   AV1LfSync lf_row_sync;
-#if CONFIG_ENTROPY
+#if CONFIG_SUBFRAME_PROB_UPDATE
   SUBFRAME_STATS subframe_stats;
   // TODO(yaowu): minimize the size of count buffers
   SUBFRAME_STATS wholeframe_stats;
   av1_coeff_stats branch_ct_buf[COEF_PROBS_BUFS][TX_SIZES][PLANE_TYPES];
-#endif  // CONFIG_ENTROPY
+#endif  // CONFIG_SUBFRAME_PROB_UPDATE
 #if CONFIG_ANS
   struct BufAnsCoder buf_ans;
 #endif

av1/encoder/subexp.c

@@ -179,7 +179,7 @@ int av1_prob_diff_update_savings_search_model(const unsigned int *ct,
   return bestsavings;
 }
 
-#if CONFIG_ENTROPY
+#if CONFIG_SUBFRAME_PROB_UPDATE
 static int get_cost(unsigned int ct[][2], aom_prob p, int n) {
   int i, p0 = p;
   unsigned int total_ct[2] = { 0, 0 };
@@ -254,7 +254,7 @@ int av1_prob_update_search_model_subframe(
   *bestp = bestnewp;
   return bestsavings;
 }
-#endif  // CONFIG_ENTROPY
+#endif  // CONFIG_SUBFRAME_PROB_UPDATE
 
 void av1_cond_prob_diff_update(aom_writer *w, aom_prob *oldp,
                                const unsigned int ct[2], int probwt) {

av1/encoder/subexp.h

@@ -35,13 +35,13 @@ int av1_prob_diff_update_savings_search_model(const unsigned int *ct,
 
 int av1_cond_prob_diff_update_savings(aom_prob *oldp, const unsigned int ct[2],
                                       int probwt);
-#if CONFIG_ENTROPY
+#if CONFIG_SUBFRAME_PROB_UPDATE
 int av1_prob_update_search_subframe(unsigned int ct[][2], aom_prob oldp,
                                     aom_prob *bestp, aom_prob upd, int n);
 int av1_prob_update_search_model_subframe(
     unsigned int ct[ENTROPY_NODES][COEF_PROBS_BUFS][2], const aom_prob *oldp,
     aom_prob *bestp, aom_prob upd, int stepsize, int n);
-#endif  // CONFIG_ENTROPY
+#endif  // CONFIG_SUBFRAME_PROB_UPDATE
 
 //
 // mag_bits is number of bits for magnitude. The alphabet is of size

av1/encoder/tokenize.c

@@ -542,14 +542,14 @@ static void tokenize_b(int plane, int block, int blk_row, int blk_col,
   unsigned int(*const counts)[COEFF_CONTEXTS][ENTROPY_TOKENS] =
       td->rd_counts.coef_counts[txsize_sqr_map[tx_size]][type][ref];
 #if !CONFIG_NEW_TOKENSET
-#if CONFIG_ENTROPY
+#if CONFIG_SUBFRAME_PROB_UPDATE
   const aom_prob(*coef_probs)[COEFF_CONTEXTS][UNCONSTRAINED_NODES] =
       cpi->subframe_stats.coef_probs_buf[cpi->common.coef_probs_update_idx]
                                         [txsize_sqr_map[tx_size]][type][ref];
 #else
   aom_prob(*const coef_probs)[COEFF_CONTEXTS][UNCONSTRAINED_NODES] =
       cpi->common.fc->coef_probs[txsize_sqr_map[tx_size]][type][ref];
-#endif  // CONFIG_ENTROPY
+#endif  // CONFIG_SUBFRAME_PROB_UPDATE
 #endif  // !CONFIG_NEW_TOKENSET
 #if CONFIG_EC_ADAPT
   FRAME_CONTEXT *ec_ctx = xd->tile_ctx;

configure

@@ -280,6 +280,8 @@ EXPERIMENT_LIST="
     ncobmc
     warped_motion
     entropy
+    q_adapt_probs
+    subframe_prob_update
     bitstream_debug
     alt_intra
     palette