From 930118c5c551025934d5e775c325b91f95f92da9 Mon Sep 17 00:00:00 2001 From: Guillaume Martres Date: Tue, 6 Dec 2016 20:26:22 -1000 Subject: [PATCH] PVQ: Fix incorrect calculation of rd_stats When PVQ is on, we reencode at the end of choose_tx_size_type_from_rd to get the entropy contexts right, previously this was done using txfm_rd_in_plane but this is different from the encodes done in the loop which use txfm_yrd, the result is that rd_stats is set incorrectly at the end of choose_tx_size_type_from_rd when PVQ is on. Results on objective-1-fast with --limit=5: PSNR | PSNR Cb | PSNR Cr | PSNR HVS | SSIM | MS SSIM | CIEDE 2000 -0.5803 | -1.0598 | -1.4565 | -0.3377 | -0.8153 | -0.5934 | -0.9943 See https://goo.gl/Hvv0E2 Change-Id: Iccc7b0afaff849f959a0084eb48dbb838bc3cb1a --- av1/encoder/rdopt.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c index 6bb84c05e..6df0926ac 100644 --- a/av1/encoder/rdopt.c +++ b/av1/encoder/rdopt.c @@ -1818,8 +1818,7 @@ static void choose_tx_size_type_from_rd(const AV1_COMP *const cpi, #endif #if CONFIG_PVQ if (best_rd != INT64_MAX) { - txfm_rd_in_plane(x, cpi, rd_stats, ref_best_rd, 0, bs, best_tx, - cpi->sf.use_fast_coef_costing); + txfm_yrd(cpi, x, rd_stats, ref_best_rd, bs, best_tx_type, best_tx); } #endif }