Verify builtin inline annotation with VM_CHECK_MODE (#3244)

* Verify builtin inline annotation with VM_CHECK_MODE

* Remove static to fix the link issue on MJIT

tool/ruby_vm/views/_insn_entry.erb

@@ -39,7 +39,7 @@ INSN_ENTRY(<%= insn.name %>)
 % if insn.handles_sp?
     POPN(INSN_ATTR(popn));
 % end
-<%= insn.handle_canary "SETUP_CANARY()" -%>
+<%= insn.handle_canary "SETUP_CANARY(leaf)" -%>
     COLLECT_USAGE_INSN(INSN_ATTR(bin));
 % insn.opes.each_with_index do |ope, i|
     COLLECT_USAGE_OPERAND(INSN_ATTR(bin), <%= i %>, <%= ope[:name] %>);
@@ -55,7 +55,7 @@ INSN_ENTRY(<%= insn.name %>)
 
     /* ### Instruction trailers. ### */
     CHECK_VM_STACK_OVERFLOW_FOR_INSN(VM_REG_CFP, INSN_ATTR(retn));
-<%= insn.handle_canary "CHECK_CANARY()" -%>
+<%= insn.handle_canary "CHECK_CANARY(leaf, INSN_ATTR(bin))" -%>
 % if insn.handles_sp?
 %   insn.rets.reverse_each do |ret|
     PUSH(<%= insn.cast_to_VALUE ret %>);

vm_insnhelper.c

@@ -4969,7 +4969,7 @@ vm_trace(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, const VALUE *p
 }
 
 #if VM_CHECK_MODE > 0
-static NORETURN( NOINLINE( COLDFUNC
+NORETURN( NOINLINE( COLDFUNC
 void vm_canary_is_found_dead(enum ruby_vminsn_type i, VALUE c)));
 
 void
@@ -5165,10 +5165,13 @@ lookup_builtin_invoker(int argc)
 }
 
 static inline VALUE
-invoke_bf(rb_execution_context_t *ec, rb_control_frame_t *cfp, const struct rb_builtin_function* bf, const VALUE *argv)
+invoke_bf(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, const struct rb_builtin_function* bf, const VALUE *argv)
 {
-    VALUE self = cfp->self;
-    return (*lookup_builtin_invoker(bf->argc))(ec, self, argv, (rb_insn_func_t)bf->func_ptr);
+    const bool canary_p = reg_cfp->iseq->body->builtin_inline_p; // Verify an assumption of `Primitive.attr! 'inline'`
+    SETUP_CANARY(canary_p);
+    VALUE ret = (*lookup_builtin_invoker(bf->argc))(ec, reg_cfp->self, argv, (rb_insn_func_t)bf->func_ptr);
+    CHECK_CANARY(canary_p, BIN(invokebuiltin));
+    return ret;
 }
 
 static VALUE

vm_insnhelper.h

@@ -139,27 +139,27 @@ CC_SET_FASTPATH(const struct rb_callcache *cc, vm_call_handler func, bool enable
 /**********************************************************/
 
 #if VM_CHECK_MODE > 0
-#define SETUP_CANARY() \
+#define SETUP_CANARY(cond) \
     VALUE *canary = 0; \
-    if (leaf) { \
+    if (cond) { \
         canary = GET_SP(); \
         SET_SV(vm_stack_canary); \
     } \
     else {\
         SET_SV(Qfalse); /* cleanup */ \
     }
-#define CHECK_CANARY() \
-    if (leaf) { \
+#define CHECK_CANARY(cond, insn) \
+    if (cond) { \
         if (*canary == vm_stack_canary) { \
             *canary = Qfalse; /* cleanup */ \
         } \
         else { \
-            vm_canary_is_found_dead(INSN_ATTR(bin), *canary); \
+            vm_canary_is_found_dead(insn, *canary); \
         } \
     }
 #else
-#define SETUP_CANARY()          /* void */
-#define CHECK_CANARY()          /* void */
+#define SETUP_CANARY(cond)       /* void */
+#define CHECK_CANARY(cond, insn) /* void */
 #endif
 
 /**********************************************************/