Clarify value ranges

This commit clarifies integer value range for vairables used in
several variance functions, also change to use proper type
conversion to reflect the value ranges.

Ported from libvpx/master: #a564b18d7fe6d348068619d2d8099cc6bf438a83

Change-Id: I94bf12a8567766003ec667b4ebe9b820596d54ca

aom_dsp/x86/halfpix_variance_sse2.c

@@ -9,6 +9,8 @@
  * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
  */
 
+#include <assert.h>
+
 #include "./aom_config.h"
 #include "./aom_dsp_rtcd.h"
 #include "aom/aom_integer.h"
@@ -38,7 +40,9 @@ uint32_t aom_variance_halfpixvar16x16_h_sse2(const unsigned char *src,
                                     &xsum0, &xxsum0);
 
   *sse = xxsum0;
-  return (xxsum0 - (((uint32_t)xsum0 * xsum0) >> 8));
+  assert(xsum0 <= 255 * 16 * 16);
+  assert(xsum0 >= -255 * 16 * 16);
+  return (xxsum0 - ((uint32_t)((int64_t)xsum0 * xsum0) >> 8));
 }
 
 uint32_t aom_variance_halfpixvar16x16_v_sse2(const unsigned char *src,
@@ -51,7 +55,9 @@ uint32_t aom_variance_halfpixvar16x16_v_sse2(const unsigned char *src,
                                    &xxsum0);
 
   *sse = xxsum0;
-  return (xxsum0 - (((uint32_t)xsum0 * xsum0) >> 8));
+  assert(xsum0 <= 255 * 16 * 16);
+  assert(xsum0 >= -255 * 16 * 16);
+  return (xxsum0 - ((uint32_t)((int64_t)xsum0 * xsum0) >> 8));
 }
 
 uint32_t aom_variance_halfpixvar16x16_hv_sse2(const unsigned char *src,
@@ -65,5 +71,7 @@ uint32_t aom_variance_halfpixvar16x16_hv_sse2(const unsigned char *src,
                                          &xsum0, &xxsum0);
 
   *sse = xxsum0;
-  return (xxsum0 - (((uint32_t)xsum0 * xsum0) >> 8));
+  assert(xsum0 <= 255 * 16 * 16);
+  assert(xsum0 >= -255 * 16 * 16);
+  return (xxsum0 - ((uint32_t)((int64_t)xsum0 * xsum0) >> 8));
 }