vp9: promote gf_group_bits calculation to 64-bit

avoids signed integer overflow

Change-Id: I9ffcdba90b21edb324d1b173fd11d613e0592931

vp9/encoder/vp9_firstpass.c

@@ -1663,8 +1663,9 @@ static void define_gf_group(VP9_COMP *cpi, FIRSTPASS_STATS *this_frame) {
 
   // Clip cpi->twopass.gf_group_bits based on user supplied data rate
   // variability limit (cpi->oxcf.two_pass_vbrmax_section)
-  if (cpi->twopass.gf_group_bits > max_bits * cpi->baseline_gf_interval)
-    cpi->twopass.gf_group_bits = max_bits * cpi->baseline_gf_interval;
+  if (cpi->twopass.gf_group_bits >
+      (int64_t)max_bits * cpi->baseline_gf_interval)
+    cpi->twopass.gf_group_bits = (int64_t)max_bits * cpi->baseline_gf_interval;
 
   // Reset the file position
   reset_fpf_position(cpi, start_pos);