From 1c9b0918b30cff580d952e52a85fb2b2a1b78d07 Mon Sep 17 00:00:00 2001 From: hui su Date: Tue, 2 Feb 2016 13:50:26 -0800 Subject: [PATCH] Fix some interger overflow errors Change-Id: I7e44bd952f28ce9925e8bdf6ee8ca2bb13de1b49 --- vp9/encoder/vp9_mcomp.c | 20 ++++++++++---------- vp9/encoder/x86/vp9_diamond_search_sad_avx.c | 5 +++-- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/vp9/encoder/vp9_mcomp.c b/vp9/encoder/vp9_mcomp.c index 4004dd3db..607941cfa 100644 --- a/vp9/encoder/vp9_mcomp.c +++ b/vp9/encoder/vp9_mcomp.c @@ -153,10 +153,10 @@ void vp9_init3smotion_compensation(search_site_config *cfg, int stride) { */ /* estimated cost of a motion vector (r,c) */ -#define MVC(r, c) \ - (mvcost ? \ - ((mvjcost[((r) != rr) * 2 + ((c) != rc)] + \ - mvcost[0][((r) - rr)] + mvcost[1][((c) - rc)]) * \ +#define MVC(r, c) \ + (mvcost ? \ + ((unsigned)(mvjcost[((r) != rr) * 2 + ((c) != rc)] + \ + mvcost[0][((r) - rr)] + mvcost[1][((c) - rc)]) * \ error_per_bit + 4096) >> 13 : 0) @@ -849,9 +849,9 @@ static INLINE void calc_int_cost_list(const MACROBLOCK *x, cost_list[i + 1] = fn_ptr->vf(what->buf, what->stride, get_buf_from_mv(in_what, &this_mv), in_what->stride, &sse) + - // mvsad_err_cost(x, &this_mv, &fcenter_mv, sadpb); - mv_err_cost(&this_mv, &fcenter_mv, x->nmvjointcost, x->mvcost, - x->errorperbit); + mv_err_cost(&this_mv, &fcenter_mv, + x->nmvjointcost, x->mvcost, + x->errorperbit); } } else { for (i = 0; i < 4; i++) { @@ -863,9 +863,9 @@ static INLINE void calc_int_cost_list(const MACROBLOCK *x, cost_list[i + 1] = fn_ptr->vf(what->buf, what->stride, get_buf_from_mv(in_what, &this_mv), in_what->stride, &sse) + - // mvsad_err_cost(x, &this_mv, &fcenter_mv, sadpb); - mv_err_cost(&this_mv, &fcenter_mv, x->nmvjointcost, x->mvcost, - x->errorperbit); + mv_err_cost(&this_mv, &fcenter_mv, + x->nmvjointcost, x->mvcost, + x->errorperbit); } } } diff --git a/vp9/encoder/x86/vp9_diamond_search_sad_avx.c b/vp9/encoder/x86/vp9_diamond_search_sad_avx.c index 2ed3f1a8b..7b28e1335 100644 --- a/vp9/encoder/x86/vp9_diamond_search_sad_avx.c +++ b/vp9/encoder/x86/vp9_diamond_search_sad_avx.c @@ -50,8 +50,9 @@ static int mvsad_err_cost(const MACROBLOCK *x, const int_mv mv, const MV *ref, int error_per_bit) { const int_mv diff = pack_int_mv(mv.as_mv.row - ref->row, mv.as_mv.col - ref->col); - return ROUND_POWER_OF_TWO(mv_cost(diff, x->nmvjointsadcost, - x->nmvsadcost) * error_per_bit, 8); + return ROUND_POWER_OF_TWO((unsigned)mv_cost(diff, x->nmvjointsadcost, + x->nmvsadcost) * + error_per_bit, 8); } /*****************************************************************************