From d43d6777a69348b8dc3a6ac58c6a8639d24b1b01 Mon Sep 17 00:00:00 2001 From: Yaowu Xu Date: Mon, 26 Jun 2017 15:48:37 -0700 Subject: [PATCH] quantize.c: convert to int before apply sign This change makes the conversions similar to those in av1_quantize.c, and fix ubsan warnings shown in nightly tests. Change-Id: I90851a80dcb9f052a32bf22199fd9ef8ff927725 --- aom_dsp/quantize.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/aom_dsp/quantize.c b/aom_dsp/quantize.c index 0759c22e3..d5c89ebf1 100644 --- a/aom_dsp/quantize.c +++ b/aom_dsp/quantize.c @@ -716,8 +716,7 @@ void aom_highbd_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, if (abs_coeff >= zbins[rc != 0]) { const int64_t tmp1 = abs_coeff + round_ptr[rc != 0]; const int64_t tmp2 = ((tmp1 * quant_ptr[rc != 0]) >> 16) + tmp1; - const uint32_t abs_qcoeff = - (uint32_t)((tmp2 * quant_shift_ptr[rc != 0]) >> 16); + const int abs_qcoeff = (int)((tmp2 * quant_shift_ptr[rc != 0]) >> 16); qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign); dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0]; if (abs_qcoeff) eob = i; @@ -767,8 +766,7 @@ void aom_highbd_quantize_b_32x32_c( const int64_t tmp1 = abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], 1); const int64_t tmp2 = ((tmp1 * quant_ptr[rc != 0]) >> 16) + tmp1; - const uint32_t abs_qcoeff = - (uint32_t)((tmp2 * quant_shift_ptr[rc != 0]) >> 15); + const int abs_qcoeff = (int)((tmp2 * quant_shift_ptr[rc != 0]) >> 15); qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign); dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0] / 2; if (abs_qcoeff) eob = idx_arr[i]; @@ -818,8 +816,7 @@ void aom_highbd_quantize_b_64x64_c( const int64_t tmp1 = abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], 2); const int64_t tmp2 = ((tmp1 * quant_ptr[rc != 0]) >> 16) + tmp1; - const uint32_t abs_qcoeff = - (uint32_t)((tmp2 * quant_shift_ptr[rc != 0]) >> 14); + const int abs_qcoeff = (int)((tmp2 * quant_shift_ptr[rc != 0]) >> 14); qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign); dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0] / 4; if (abs_qcoeff) eob = idx_arr[i];