variable.c: check index overflow

* variable.c (rb_ivar_set), vm_insnhelper.c (vm_setivar): check
  instance variable index overflow.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@47512 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

variable.c

@@ -1153,6 +1153,7 @@ rb_ivar_set(VALUE obj, ID id, VALUE val)
         ivar_extended = 0;
         if (!st_lookup(iv_index_tbl, (st_data_t)id, &index)) {
             index = iv_index_tbl->num_entries;
+	    if (index >= INT_MAX) rb_raise(rb_eArgError, "too many instance variables");
             st_add_direct(iv_index_tbl, (st_data_t)id, index);
             ivar_extended = 1;
         }

vm_insnhelper.c

@@ -570,8 +570,11 @@ vm_setivar(VALUE obj, ID id, VALUE val, IC ic, rb_call_info_t *ci, int is_attr)
 		    ic->ic_value.index = index;
 		    ic->ic_serial = RCLASS_SERIAL(klass);
 		}
+		else if (index >= INT_MAX) {
+		    rb_raise(rb_eArgError, "too many instance variables");
+		}
 		else {
-		    ci->aux.index = index + 1;
+		    ci->aux.index = (int)(index + 1);
 		}
 	    }
 	    /* fall through */