array.c: fix array size overflow

* array.c (ary_ensure_room_for_push): check if array size will
  exceed maxmum size to get rid of buffer overflow.
  [ruby-dev:49043] [Bug #11235]
* array.c (ary_ensure_room_for_unshift, rb_ary_splice): ditto.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@50827 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

ChangeLog

@@ -1,3 +1,11 @@
+Thu Jun 11 13:50:19 2015  Nobuyoshi Nakada  <nobu@ruby-lang.org>
+
+	* array.c (ary_ensure_room_for_push): check if array size will
+	  exceed maxmum size to get rid of buffer overflow.
+	  [ruby-dev:49043] [Bug #11235]
+
+	* array.c (ary_ensure_room_for_unshift, rb_ary_splice): ditto.
+
 Thu Jun 11 13:17:34 2015  NAKAMURA Usaku  <usa@ruby-lang.org>
 
 	* test/test_cmath.rb (TestCMath#test_trigonometric_functions): should

array.c

@@ -353,9 +353,13 @@ rb_ary_modify(VALUE ary)
 static VALUE
 ary_ensure_room_for_push(VALUE ary, long add_len)
 {
-    long new_len = RARRAY_LEN(ary) + add_len;
+    long old_len = RARRAY_LEN(ary);
+    long new_len = old_len + add_len;
     long capa;
 
+    if (old_len > ARY_MAX_SIZE - add_len) {
+	rb_raise(rb_eIndexError, "index %ld too big", new_len);
+    }
     if (ARY_SHARED_P(ary)) {
 	if (new_len > RARRAY_EMBED_LEN_MAX) {
 	    VALUE shared = ARY_SHARED(ary);
@@ -1095,6 +1099,10 @@ ary_ensure_room_for_unshift(VALUE ary, int argc)
     long capa;
     const VALUE *head, *sharedp;
 
+    if (len > ARY_MAX_SIZE - argc) {
+	rb_raise(rb_eIndexError, "index %ld too big", new_len);
+    }
+
     if (ARY_SHARED_P(ary)) {
 	VALUE shared = ARY_SHARED(ary);
 	capa = RARRAY_LEN(shared);
@@ -1592,6 +1600,9 @@ rb_ary_splice(VALUE ary, long beg, long len, VALUE rpl)
     else {
 	long alen;
 
+	if (olen - len > ARY_MAX_SIZE - rlen) {
+	    rb_raise(rb_eIndexError, "index %ld too big", olen + rlen - len);
+	}
 	rb_ary_modify(ary);
 	alen = olen + rlen - len;
 	if (alen >= ARY_CAPA(ary)) {

test/ruby/test_array.rb

@@ -2553,6 +2553,34 @@ class TestArray < Test::Unit::TestCase
     end
   end
 
+  sizeof_long = [0].pack("l!").size
+  sizeof_voidp = [""].pack("p").size
+  if sizeof_long < sizeof_voidp
+    ARY_MAX = (1<<(8*sizeof_long-1)) / sizeof_voidp - 1
+    Bug11235 = '[ruby-dev:49043] [Bug #11235]'
+
+    def test_push_over_ary_max
+      assert_separately(['-', ARY_MAX.to_s, Bug11235], <<-"end;")
+        a = Array.new(ARGV[0].to_i)
+        assert_raise(IndexError, ARGV[1]) {0x1000.times {a.push(1)}}
+      end;
+    end
+
+    def test_unshift_over_ary_max
+      assert_separately(['-', ARY_MAX.to_s, Bug11235], <<-"end;")
+        a = Array.new(ARGV[0].to_i)
+        assert_raise(IndexError, ARGV[1]) {0x1000.times {a.unshift(1)}}
+      end;
+    end
+
+    def test_splice_over_ary_max
+      assert_separately(['-', ARY_MAX.to_s, Bug11235], <<-"end;")
+        a = Array.new(ARGV[0].to_i)
+        assert_raise(IndexError, ARGV[1]) {a[0, 0] = Array.new(0x1000)}
+      end;
+    end
+  end
+
   private
   def need_continuation
     unless respond_to?(:callcc, true)