Fix stack buffer overflow

https://hackerone.com/reports/1306859

include/ruby/internal/memory.h

@@ -284,7 +284,7 @@ typedef uint128_t DSIZE_T;
  * @return      A pointer on stack.
  */
 #define ALLOCA_N(type,n) \
-    RBIMPL_CAST((type *)alloca(rbimpl_size_mul_or_raise(sizeof(type), (n))))
+    RBIMPL_CAST((type *)(!(n) ? NULL : alloca(rbimpl_size_mul_or_raise(sizeof(type), (n)))))
 
 /**
  * Identical to #RB_ALLOCV_N(), except it  implicitly assumes the type of array
@@ -297,7 +297,7 @@ typedef uint128_t DSIZE_T;
  */
 #define RB_ALLOCV(v, n)        \
     ((n) < RUBY_ALLOCV_LIMIT ? \
-     ((v) = 0, alloca(n)) :    \
+     ((v) = 0, !(n) ? NULL : alloca(n)) : \
      rb_alloc_tmp_buffer(&(v), (n)))
 
 /**
@@ -330,7 +330,7 @@ typedef uint128_t DSIZE_T;
 #define RB_ALLOCV_N(type, v, n)                             \
     RBIMPL_CAST((type *)                                     \
         (((size_t)(n) < RUBY_ALLOCV_LIMIT / sizeof(type)) ? \
-         ((v) = 0, alloca((n) * sizeof(type))) :            \
+         ((v) = 0, !(n) ? NULL : alloca((n) * sizeof(type))) : \
          rb_alloc_tmp_buffer2(&(v), (n), sizeof(type))))
 
 /**

random.c

@@ -365,15 +365,12 @@ rand_init(const rb_random_interface_t *rng, rb_random_t *rnd, VALUE seed)
     int sign;
 
     len = rb_absint_numwords(seed, 32, NULL);
+    if (len == 0) len = 1;
     buf = ALLOCV_N(uint32_t, buf0, len);
     sign = rb_integer_pack(seed, buf, len, sizeof(uint32_t), 0,
         INTEGER_PACK_LSWORD_FIRST|INTEGER_PACK_NATIVE_BYTE_ORDER);
     if (sign < 0)
         sign = -sign;
-    if (len == 0) {
-        buf[0] = 0;
-        len = 1;
-    }
     if (len > 1) {
         if (sign != 2 && buf[len-1] == 1) /* remove leading-zero-guard */
             len--;
@@ -883,7 +880,7 @@ rand_mt_init(rb_random_t *rnd, const uint32_t *buf, size_t len)
 {
     struct MT *mt = &((rb_random_mt_t *)rnd)->mt;
     if (len <= 1) {
-        init_genrand(mt, buf[0]);
+        init_genrand(mt, len ? buf[0] : 0);
     }
     else {
         init_by_array(mt, buf, (int)len);