[Bug #19748] Fix out-of-bound access in `String#byteindex`

string.c

@@ -3970,20 +3970,21 @@ rb_str_byteindex_m(int argc, VALUE *argv, VALUE str)
     long pos;
 
     if (rb_scan_args(argc, argv, "11", &sub, &initpos) == 2) {
+        long slen = RSTRING_LEN(str);
         pos = NUM2LONG(initpos);
-    }
-    else {
-        pos = 0;
-    }
-    if (pos < 0) {
-        pos += RSTRING_LEN(str);
         if (pos < 0) {
+            pos += slen;
+        }
+        if (pos < 0 || pos > slen) {
             if (RB_TYPE_P(sub, T_REGEXP)) {
                 rb_backref_set(Qnil);
             }
             return Qnil;
         }
     }
+    else {
+        pos = 0;
+    }
 
     if (!str_check_byte_pos(str, pos)) {
         rb_raise(rb_eIndexError,
@@ -3991,10 +3992,6 @@ rb_str_byteindex_m(int argc, VALUE *argv, VALUE str)
     }
 
     if (RB_TYPE_P(sub, T_REGEXP)) {
-        if (pos > RSTRING_LEN(str)) {
-            rb_backref_set(Qnil);
-            return Qnil;
-        }
         if (rb_reg_search(sub, str, pos, 0) < 0) {
             return Qnil;
         }

test/ruby/test_string.rb

@@ -3397,6 +3397,9 @@ CODE
     assert_byteindex(6, S("にんにちは"), S("に"), 6)
     assert_byteindex(6, S("にんにちは"), /に./, 6)
     assert_raise(IndexError) { S("にんにちは").byteindex(?に, 7) }
+
+    s = S("foobarbarbaz")
+    assert !1000.times.any? {s.byteindex("", 100_000_000)}
   end
 
   def test_byterindex