vm_insnhelper.c: recv -1 + 3 overflows

Here, recv can be INT2FIX(-1), which is 0xFFFF_FFFFul.
INT2FIX(1) is 3ul.  So `recv - 1 + INT2FIX(1)` is:

recv              0xFFFF_FFFFul
recv-1            0xFFFF_FFFEul (note: unsigned)
recv-1+INT2FIX(1) 0x0000_0001ul Here is the overflow.

Given recv is a Fixnum, it can never be 0xFFFF_FFFD.  0xFFFF_FFFF is
the only value that can overflow this way, so special-casing this
value should just suffice.



git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@65828 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

vm_insnhelper.c

@@ -3762,18 +3762,40 @@ vm_opt_empty_p(VALUE recv)
     }
 }
 
+static VALUE
+fix_succ(VALUE x)
+{
+    switch (x) {
+    case ~0UL:
+      /* 0xFFFF_FFFF == INT2FIX(-1)
+       * `-1.succ` is of course 0. */
+      return INT2FIX(0);
+    case RSHIFT(~0UL, 1):
+      /* 0x7FFF_FFFF == LONG2FIX(0x3FFF_FFFF)
+       * 0x3FFF_FFFF + 1 == 0x4000_0000, which is a Bignum. */
+      return rb_uint2big(1UL << (SIZEOF_LONG * CHAR_BIT - 2));
+    default:
+      /*    LONG2FIX(FIX2LONG(x)+FIX2LONG(y))
+       * == ((lx*2+1)/2 + (ly*2+1)/2)*2+1
+       * == lx*2 + ly*2 + 1
+       * == (lx*2+1) + (ly*2+1) - 1
+       * == x + y - 1
+       *
+       * Here, if we put y := INT2FIX(1):
+       *
+       * == x + INT2FIX(1) - 1
+       * == x + 2 .
+       */
+      return x + 2;
+    }
+}
+
 static VALUE
 vm_opt_succ(VALUE recv)
 {
     if (FIXNUM_P(recv) &&
 	BASIC_OP_UNREDEFINED_P(BOP_SUCC, INTEGER_REDEFINED_OP_FLAG)) {
-	/* fixnum + INT2FIX(1) */
-	if (recv == LONG2FIX(FIXNUM_MAX)) {
-	    return LONG2NUM(FIXNUM_MAX + 1);
-	}
-	else {
-	    return recv - 1 + INT2FIX(1);
-	}
+        return fix_succ(recv);
     }
     else if (SPECIAL_CONST_P(recv)) {
 	return Qundef;