* gc.c (objspace_malloc_prepare): remove size check because it is

used by objspace_xmalloc and objspace_xcalloc.
  objspace_xmalloc introduces its own check in this commit.
  objspace_xcalloc checks with xmalloc2_size (ruby_xmalloc2_size).

* gc.c (objspace_xmalloc0): common xmalloc function.

* gc.c (objspace_xmalloc): introduce its own size check.

* gc.c (objspace_xmalloc2): separated from ruby_xmalloc2 to clarify
  the layer who has the responsibility to check the size.

* gc.c (objspace_xrealloc): remove duplicated size check.

* gc.c (ruby_xmalloc2): use objspace_xmalloc2.

* include/ruby/ruby.h (ruby_xmalloc2_size): follow the size limit
  as SSIZE_MAX. Note that ISO C says size_t is unsigned integer.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@54661 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

ChangeLog

@@ -1,3 +1,24 @@
+Thu Apr 21 01:44:19 2016  NARUSE, Yui  <naruse@ruby-lang.org>
+
+	* gc.c (objspace_malloc_prepare): remove size check because it is
+	  used by objspace_xmalloc and objspace_xcalloc.
+	  objspace_xmalloc introduces its own check in this commit.
+	  objspace_xcalloc checks with xmalloc2_size (ruby_xmalloc2_size).
+
+	* gc.c (objspace_xmalloc0): common xmalloc function.
+
+	* gc.c (objspace_xmalloc): introduce its own size check.
+
+	* gc.c (objspace_xmalloc2): separated from ruby_xmalloc2 to clarify
+	  the layer who has the responsibility to check the size.
+
+	* gc.c (objspace_xrealloc): remove duplicated size check.
+
+	* gc.c (ruby_xmalloc2): use objspace_xmalloc2.
+
+	* include/ruby/ruby.h (ruby_xmalloc2_size): follow the size limit
+	  as SSIZE_MAX. Note that ISO C says size_t is unsigned integer.
+
 Thu Apr 21 12:14:04 2016  Nobuyoshi Nakada  <nobu@ruby-lang.org>
 
 	* configure.in: check if succeeded in creating config.h.

gc.c

@@ -7739,9 +7739,6 @@ objspace_malloc_increase(rb_objspace_t *objspace, void *mem, size_t new_size, si
 static inline size_t
 objspace_malloc_prepare(rb_objspace_t *objspace, size_t size)
 {
-    if ((ssize_t)size < 0) {
-	negative_size_allocation_error("negative allocation size (or too big)");
-    }
     if (size == 0) size = 1;
 
 #if CALC_EXACT_MALLOC_SIZE
@@ -7771,8 +7768,11 @@ objspace_malloc_fixup(rb_objspace_t *objspace, void *mem, size_t size)
 	} \
     } while (0)
 
+/* this shouldn't be called directly.
+ * objspace_xmalloc and objspace_xmalloc2 checks allocation size.
+ */
 static void *
-objspace_xmalloc(rb_objspace_t *objspace, size_t size)
+objspace_xmalloc0(rb_objspace_t *objspace, size_t size)
 {
     void *mem;
 
@@ -7783,15 +7783,27 @@ objspace_xmalloc(rb_objspace_t *objspace, size_t size)
     return objspace_malloc_fixup(objspace, mem, size);
 }
 
+static void *
+objspace_xmalloc(rb_objspace_t *objspace, size_t size)
+{
+    if ((ssize_t)size < 0) {
+	negative_size_allocation_error("too large allocation size");
+    }
+    return objspace_xmalloc0(objspace, size);
+}
+
+#define xmalloc2_size ruby_xmalloc2_size
+static void *
+objspace_xmalloc2(rb_objspace_t *objspace, size_t n, size_t size)
+{
+    return objspace_xmalloc0(&rb_objspace, xmalloc2_size(n, size));
+}
+
 static void *
 objspace_xrealloc(rb_objspace_t *objspace, void *ptr, size_t new_size, size_t old_size)
 {
     void *mem;
 
-    if ((ssize_t)new_size < 0) {
-	negative_size_allocation_error("negative re-allocation size");
-    }
-
     if (!ptr) return objspace_xmalloc(objspace, new_size);
 
     /*
@@ -7852,12 +7864,10 @@ ruby_malloc_size_overflow(size_t count, size_t elsize)
 	     count, elsize);
 }
 
-#define xmalloc2_size ruby_xmalloc2_size
-
 void *
 ruby_xmalloc2(size_t n, size_t size)
 {
-    return objspace_xmalloc(&rb_objspace, xmalloc2_size(n, size));
+    return objspace_xmalloc2(&rb_objspace, n, size);
 }
 
 static void *

include/ruby/ruby.h

@@ -1611,7 +1611,7 @@ NORETURN(void ruby_malloc_size_overflow(size_t, size_t));
 static inline size_t
 ruby_xmalloc2_size(const size_t count, const size_t elsize)
 {
-    if (count > SIZE_MAX / elsize) {
+    if (count > SSIZE_MAX / elsize) {
 	ruby_malloc_size_overflow(count, elsize);
     }
     return count * elsize;