diff --git a/av1/common/blockd.h b/av1/common/blockd.h index 94eb089b1..266f8fe62 100644 --- a/av1/common/blockd.h +++ b/av1/common/blockd.h @@ -516,8 +516,10 @@ static INLINE int is_rect_tx_allowed_bsize(BLOCK_SIZE bsize) { return LUT[bsize]; } -static INLINE int is_rect_tx_allowed(const MB_MODE_INFO *mbmi) { - return is_inter_block(mbmi) && is_rect_tx_allowed_bsize(mbmi->sb_type); +static INLINE int is_rect_tx_allowed(const MACROBLOCKD *xd, + const MB_MODE_INFO *mbmi) { + return is_inter_block(mbmi) && is_rect_tx_allowed_bsize(mbmi->sb_type) && + !xd->lossless[mbmi->segment_id]; } static INLINE int is_rect_tx(TX_SIZE tx_size) { return tx_size >= TX_SIZES; } diff --git a/av1/decoder/decodemv.c b/av1/decoder/decodemv.c index 6492f6209..7be3e752a 100644 --- a/av1/decoder/decodemv.c +++ b/av1/decoder/decodemv.c @@ -1633,7 +1633,8 @@ static void read_inter_frame_mode_info(AV1Decoder *const pbi, int idx, idy; int tx_size_cat = inter_tx_size_cat_lookup[bsize]; #if CONFIG_EXT_TX && CONFIG_RECT_TX - int is_rect_tx_allowed = inter_block && is_rect_tx_allowed_bsize(bsize); + int is_rect_tx_allowed = inter_block && is_rect_tx_allowed_bsize(bsize) && + !xd->lossless[mbmi->segment_id]; int use_rect_tx = 0; if (is_rect_tx_allowed) { diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c index 763615184..f9ae15466 100644 --- a/av1/encoder/bitstream.c +++ b/av1/encoder/bitstream.c @@ -406,7 +406,7 @@ static void write_selected_tx_size(const AV1_COMMON *cm, const MACROBLOCKD *xd, const TX_SIZE coded_tx_size = txsize_sqr_up_map[tx_size]; #if CONFIG_EXT_TX && CONFIG_RECT_TX - assert(IMPLIES(is_rect_tx(tx_size), is_rect_tx_allowed(mbmi))); + assert(IMPLIES(is_rect_tx(tx_size), is_rect_tx_allowed(xd, mbmi))); assert( IMPLIES(is_rect_tx(tx_size), tx_size == max_txsize_rect_lookup[bsize])); #endif // CONFIG_EXT_TX && CONFIG_RECT_TX @@ -1132,7 +1132,7 @@ static void pack_inter_mode_mvs(AV1_COMP *cpi, const MODE_INFO *mi, int idx, idy; #if CONFIG_EXT_TX && CONFIG_RECT_TX - if (is_rect_tx_allowed(mbmi)) { + if (is_rect_tx_allowed(xd, mbmi)) { int tx_size_cat = inter_tx_size_cat_lookup[bsize]; aom_write(w, is_rect_tx(mbmi->tx_size), diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c index cfc47188a..cc73d4c3f 100644 --- a/av1/encoder/encodeframe.c +++ b/av1/encoder/encodeframe.c @@ -5141,14 +5141,14 @@ static void encode_superblock(AV1_COMP *cpi, ThreadData *td, TOKENEXTRA **t, : intra_tx_size_cat_lookup[bsize]; const TX_SIZE coded_tx_size = txsize_sqr_up_map[mbmi->tx_size]; #if CONFIG_EXT_TX && CONFIG_RECT_TX - assert(IMPLIES(is_rect_tx(mbmi->tx_size), is_rect_tx_allowed(mbmi))); + assert(IMPLIES(is_rect_tx(mbmi->tx_size), is_rect_tx_allowed(xd, mbmi))); #endif // CONFIG_EXT_TX && CONFIG_RECT_TX #if CONFIG_VAR_TX #if CONFIG_EXT_TX && CONFIG_RECT_TX - if (is_rect_tx_allowed(mbmi)) { + if (is_rect_tx_allowed(xd, mbmi)) { td->counts->rect_tx[tx_size_cat][is_rect_tx(mbmi->tx_size)]++; } - if (!is_rect_tx_allowed(mbmi) || !is_rect_tx(mbmi->tx_size)) { + if (!is_rect_tx_allowed(xd, mbmi) || !is_rect_tx(mbmi->tx_size)) { #endif // CONFIG_EXT_TX && CONFIG_RECT_TX if (is_inter) tx_partition_count_update(cm, xd, bsize, mi_row, mi_col, td->counts); diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c index 9721abeb0..8b0a7bdca 100644 --- a/av1/encoder/rdopt.c +++ b/av1/encoder/rdopt.c @@ -1385,14 +1385,14 @@ static int64_t choose_tx_size_fix_type(AV1_COMP *cpi, BLOCK_SIZE bs, const int is_inter = is_inter_block(mbmi); #if CONFIG_EXT_TX #if CONFIG_RECT_TX - int evaulate_rect_tx = 0; + int evaluate_rect_tx = 0; #endif // CONFIG_RECT_TX int ext_tx_set; #endif // CONFIG_EXT_TX if (tx_select) { #if CONFIG_EXT_TX && CONFIG_RECT_TX - evaulate_rect_tx = is_rect_tx_allowed(mbmi); + evaluate_rect_tx = is_rect_tx_allowed(xd, mbmi); #endif // CONFIG_EXT_TX && CONFIG_RECT_TX start_tx = max_tx_size; end_tx = 0; @@ -1400,8 +1400,8 @@ static int64_t choose_tx_size_fix_type(AV1_COMP *cpi, BLOCK_SIZE bs, const TX_SIZE chosen_tx_size = tx_size_from_tx_mode(bs, cm->tx_mode, is_inter); #if CONFIG_EXT_TX && CONFIG_RECT_TX - evaulate_rect_tx = is_rect_tx(chosen_tx_size); - assert(IMPLIES(evaulate_rect_tx, is_rect_tx_allowed(mbmi))); + evaluate_rect_tx = is_rect_tx(chosen_tx_size); + assert(IMPLIES(evaluate_rect_tx, is_rect_tx_allowed(xd, mbmi))); #endif // CONFIG_EXT_TX && CONFIG_RECT_TX start_tx = chosen_tx_size; end_tx = chosen_tx_size; @@ -1415,7 +1415,7 @@ static int64_t choose_tx_size_fix_type(AV1_COMP *cpi, BLOCK_SIZE bs, mbmi->tx_type = tx_type; #if CONFIG_EXT_TX && CONFIG_RECT_TX - if (evaulate_rect_tx) { + if (evaluate_rect_tx) { const TX_SIZE rect_tx_size = max_txsize_rect_lookup[bs]; const int ext_tx_set = get_ext_tx_set(rect_tx_size, bs, 1); if (ext_tx_used_inter[ext_tx_set][tx_type]) { @@ -3214,7 +3214,7 @@ static int64_t select_tx_size_fix_type(const AV1_COMP *cpi, MACROBLOCK *x, mbmi->tx_type = tx_type; inter_block_yrd(cpi, x, rate, dist, skippable, sse, bsize, ref_best_rd); #if CONFIG_EXT_TX && CONFIG_RECT_TX - if (is_rect_tx_allowed(mbmi)) { + if (is_rect_tx_allowed(xd, mbmi)) { int rate_rect_tx, skippable_rect_tx = 0; int64_t dist_rect_tx, sse_rect_tx, rd, rd_rect_tx; int tx_size_cat = inter_tx_size_cat_lookup[bsize];