From 57d92577d4e87e16d9e625a72117a0c9da476124 Mon Sep 17 00:00:00 2001 From: Yaowu Xu Date: Wed, 18 May 2016 09:06:52 -0700 Subject: [PATCH] change to use correct type This commit changes to use uint32_t for cost (always non-negative), and promote to int64_t before calculation of the savings. This fixes an integer overflow. cherry-picked #a3028ddf from aom/master Change-Id: I71c2580d188cc79d2d8069241d0353cf331b5c83 --- av1/encoder/subexp.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/av1/encoder/subexp.c b/av1/encoder/subexp.c index 6ba20044f..84d748e62 100644 --- a/av1/encoder/subexp.c +++ b/av1/encoder/subexp.c @@ -117,15 +117,16 @@ void av1_write_prob_diff_update(aom_writer *w, aom_prob newp, aom_prob oldp) { int av1_prob_diff_update_savings_search(const unsigned int *ct, aom_prob oldp, aom_prob *bestp, aom_prob upd) { - const int old_b = cost_branch256(ct, oldp); + const uint32_t old_b = cost_branch256(ct, oldp); int bestsavings = 0; aom_prob newp, bestnewp = oldp; const int step = *bestp > oldp ? -1 : 1; for (newp = *bestp; newp != oldp; newp += step) { - const int new_b = cost_branch256(ct, newp); - const int update_b = prob_diff_update_cost(newp, oldp) + av1_cost_upd256; - const int savings = old_b - new_b - update_b; + const uint32_t new_b = cost_branch256(ct, newp); + const uint32_t update_b = + prob_diff_update_cost(newp, oldp) + av1_cost_upd256; + const int savings = (int)((int64_t)old_b - new_b - update_b); if (savings > bestsavings) { bestsavings = savings; bestnewp = newp;