string.c: integer overflow

* string.c (rb_str_modify_expand): check integer overflow.
  [ruby-core:75592] [Bug #12390]

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@55054 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

ChangeLog

@@ -1,3 +1,8 @@
+Wed May 18 14:52:38 2016  Nobuyoshi Nakada  <nobu@ruby-lang.org>
+
+	* string.c (rb_str_modify_expand): check integer overflow.
+	  [ruby-core:75592] [Bug #12390]
+
 Wed May 18 13:11:44 2016  NARUSE, Yui  <naruse@ruby-lang.org>
 
 	* re.c (match_ary_subseq): get subseq of match array without creating

string.c

@@ -1914,6 +1914,9 @@ rb_str_modify_expand(VALUE str, long expand)
     else if (expand > 0) {
 	long len = RSTRING_LEN(str);
 	long capa = len + expand;
+	if (expand >= LONG_MAX - len - termlen) {
+	    rb_raise(rb_eArgError, "string size too big");
+	}
 	if (!STR_EMBED_P(str)) {
 	    REALLOC_N(RSTRING(str)->as.heap.ptr, char, capa + termlen);
 	    RSTRING(str)->as.heap.aux.capa = capa;

test/-ext-/string/test_modify_expand.rb

@@ -13,4 +13,13 @@ class Test_StringModifyExpand < Test::Unit::TestCase
       s.replace("")
     CMD
   end
+
+  def test_integer_overflow
+    bug12390 = '[ruby-core:75592] [Bug #12390]'
+    s = Bug::String.new
+    long_max = (1 << (8 * RbConfig::SIZEOF['long'] - 1)) - 1
+    assert_raise(ArgumentError, bug12390) {
+      s.modify_expand!(long_max)
+    }
+  end
 end