mozilla
/
aom
зеркало из
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

quantize.c: convert to int before apply sign 

This change makes the conversions similar to those in av1_quantize.c,
and fix ubsan warnings shown in nightly tests.

Change-Id: I90851a80dcb9f052a32bf22199fd9ef8ff927725
This commit is contained in:
Yaowu Xu 2017-06-26 15:48:37 -07:00
Родитель 284c883029
Коммит d43d6777a6
1 изменённых файлов: 3 добавлений и 6 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

9
aom_dsp/quantize.c
Просмотреть файл

@ -716,8 +716,7 @@ void aom_highbd_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
if (abs_coeff >= zbins[rc != 0]) { if (abs_coeff >= zbins[rc != 0]) {
const int64_t tmp1 = abs_coeff + round_ptr[rc != 0]; const int64_t tmp1 = abs_coeff + round_ptr[rc != 0];
const int64_t tmp2 = ((tmp1 * quant_ptr[rc != 0]) >> 16) + tmp1; const int64_t tmp2 = ((tmp1 * quant_ptr[rc != 0]) >> 16) + tmp1;
const uint32_t abs_qcoeff = const int abs_qcoeff = (int)((tmp2 * quant_shift_ptr[rc != 0]) >> 16);
(uint32_t)((tmp2 * quant_shift_ptr[rc != 0]) >> 16);
qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign); qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign);
dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0]; dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0];
if (abs_qcoeff) eob = i; if (abs_qcoeff) eob = i;
@ -767,8 +766,7 @@ void aom_highbd_quantize_b_32x32_c(
const int64_t tmp1 = const int64_t tmp1 =
abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], 1); abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], 1);
const int64_t tmp2 = ((tmp1 * quant_ptr[rc != 0]) >> 16) + tmp1; const int64_t tmp2 = ((tmp1 * quant_ptr[rc != 0]) >> 16) + tmp1;
const uint32_t abs_qcoeff = const int abs_qcoeff = (int)((tmp2 * quant_shift_ptr[rc != 0]) >> 15);
(uint32_t)((tmp2 * quant_shift_ptr[rc != 0]) >> 15);
qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign); qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign);
dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0] / 2; dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0] / 2;
if (abs_qcoeff) eob = idx_arr[i]; if (abs_qcoeff) eob = idx_arr[i];
@ -818,8 +816,7 @@ void aom_highbd_quantize_b_64x64_c(
const int64_t tmp1 = const int64_t tmp1 =
abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], 2); abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], 2);
const int64_t tmp2 = ((tmp1 * quant_ptr[rc != 0]) >> 16) + tmp1; const int64_t tmp2 = ((tmp1 * quant_ptr[rc != 0]) >> 16) + tmp1;
const uint32_t abs_qcoeff = const int abs_qcoeff = (int)((tmp2 * quant_shift_ptr[rc != 0]) >> 14);
(uint32_t)((tmp2 * quant_shift_ptr[rc != 0]) >> 14);
qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign); qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign);
dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0] / 4; dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0] / 4;
if (abs_qcoeff) eob = idx_arr[i]; if (abs_qcoeff) eob = idx_arr[i];