From 280419d0e0ba3e96e19551c70cba789fbedd80e1 Mon Sep 17 00:00:00 2001 From: Koichi Sasada Date: Mon, 31 Jul 2023 16:04:16 +0900 Subject: [PATCH] `calling->cd` instead of `calling->ci` `struct rb_calling_info::cd` is introduced and `rb_calling_info::ci` is replaced with it to manipulate the inline cache of iseq while method invocation process. So that `ci` can be acessed with `calling->cd->ci`. It adds one indirection but it can be justified by the following points: 1) `vm_search_method_fastpath()` doesn't need `ci` and also `vm_call_iseq_setup_normal()` doesn't need `ci`. It means reducing `cd->ci` access in `vm_sendish()` can make it faster. 2) most of method types need to access `ci` once in theory so that 1 additional indirection doesn't matter. --- vm_core.h | 2 +- vm_eval.c | 11 +++++--- vm_insnhelper.c | 67 +++++++++++++++++++++++++++---------------------- vm_method.c | 1 + 4 files changed, 46 insertions(+), 35 deletions(-) diff --git a/vm_core.h b/vm_core.h index 64ea673d77..77861372ff 100644 --- a/vm_core.h +++ b/vm_core.h @@ -278,7 +278,7 @@ union iseq_inline_storage_entry { }; struct rb_calling_info { - const struct rb_callinfo *ci; + const struct rb_call_data *cd; const struct rb_callcache *cc; VALUE block_handler; VALUE recv; diff --git a/vm_eval.c b/vm_eval.c index 9f72d48709..fb24e2a9e9 100644 --- a/vm_eval.c +++ b/vm_eval.c @@ -96,7 +96,10 @@ vm_call0_cc(rb_execution_context_t *ec, VALUE recv, ID id, int argc, const VALUE } struct rb_calling_info calling = { - .ci = &VM_CI_ON_STACK(id, flags, argc, NULL), + .cd = &(struct rb_call_data) { + .ci = &VM_CI_ON_STACK(id, flags, argc, NULL), + .cc = NULL, + }, .cc = cc, .block_handler = vm_passed_block_handler(ec), .recv = recv, @@ -117,7 +120,7 @@ vm_call0_cme(rb_execution_context_t *ec, struct rb_calling_info *calling, const static VALUE vm_call0_super(rb_execution_context_t *ec, struct rb_calling_info *calling, const VALUE *argv, VALUE klass, enum method_missing_reason ex) { - ID mid = vm_ci_mid(calling->ci); + ID mid = vm_ci_mid(calling->cd->ci); klass = RCLASS_SUPER(klass); if (klass) { @@ -136,7 +139,7 @@ vm_call0_super(rb_execution_context_t *ec, struct rb_calling_info *calling, cons static VALUE vm_call0_cfunc_with_frame(rb_execution_context_t* ec, struct rb_calling_info *calling, const VALUE *argv) { - const struct rb_callinfo *ci = calling->ci; + const struct rb_callinfo *ci = calling->cd->ci; VALUE val; const rb_callable_method_entry_t *me = vm_cc_cme(calling->cc); const rb_method_cfunc_t *cfunc = UNALIGNED_MEMBER_PTR(me->def, body.cfunc); @@ -201,7 +204,7 @@ vm_call_check_arity(struct rb_calling_info *calling, int argc, const VALUE *argv static VALUE vm_call0_body(rb_execution_context_t *ec, struct rb_calling_info *calling, const VALUE *argv) { - const struct rb_callinfo *ci = calling->ci; + const struct rb_callinfo *ci = calling->cd->ci; const struct rb_callcache *cc = calling->cc; VALUE ret; diff --git a/vm_insnhelper.c b/vm_insnhelper.c index 22373fb82f..5e52207d81 100644 --- a/vm_insnhelper.c +++ b/vm_insnhelper.c @@ -2142,7 +2142,9 @@ vm_search_method_slowpath0(VALUE cd_owner, struct rb_call_data *cd, VALUE klass) cd->cc = cc; const struct rb_callcache *empty_cc = &vm_empty_cc; - if (cd_owner && cc != empty_cc) RB_OBJ_WRITTEN(cd_owner, Qundef, cc); + if (cd_owner && cc != empty_cc) { + RB_OBJ_WRITTEN(cd_owner, Qundef, cc); + } #if USE_DEBUG_COUNTER if (old_cc == empty_cc) { @@ -2836,7 +2838,7 @@ static VALUE vm_call_iseq_setup_kwparm_kwarg(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling) { - const struct rb_callinfo *ci = calling->ci; + const struct rb_callinfo *ci = calling->cd->ci; const struct rb_callcache *cc = calling->cc; VM_ASSERT(vm_ci_flag(ci) & VM_CALL_KWARG); @@ -2863,7 +2865,7 @@ static VALUE vm_call_iseq_setup_kwparm_nokwarg(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling) { - const struct rb_callinfo *MAYBE_UNUSED(ci) = calling->ci; + const struct rb_callinfo *MAYBE_UNUSED(ci) = calling->cd->ci; const struct rb_callcache *cc = calling->cc; VM_ASSERT((vm_ci_flag(ci) & VM_CALL_KWARG) == 0); @@ -2903,7 +2905,7 @@ static inline int vm_callee_setup_arg(rb_execution_context_t *ec, struct rb_calling_info *calling, const rb_iseq_t *iseq, VALUE *argv, int param_size, int local_size) { - const struct rb_callinfo *ci = calling->ci; + const struct rb_callinfo *ci = calling->cd->ci; const struct rb_callcache *cc = calling->cc; bool cacheable_ci = vm_ci_markable(ci); @@ -2917,7 +2919,7 @@ vm_callee_setup_arg(rb_execution_context_t *ec, struct rb_calling_info *calling, argument_arity_error(ec, iseq, calling->argc, lead_num, lead_num); } - VM_ASSERT(ci == calling->ci); + VM_ASSERT(ci == calling->cd->ci); VM_ASSERT(cc == calling->cc); if (cacheable_ci && vm_call_iseq_optimizable_p(ci, cc)) { @@ -3024,7 +3026,7 @@ static inline VALUE vm_call_iseq_setup_2(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling, int opt_pc, int param_size, int local_size) { - const struct rb_callinfo *ci = calling->ci; + const struct rb_callinfo *ci = calling->cd->ci; const struct rb_callcache *cc = calling->cc; if (LIKELY(!(vm_ci_flag(ci) & VM_CALL_TAILCALL))) { @@ -3428,7 +3430,7 @@ vm_call_cfunc_with_frame_(rb_execution_context_t *ec, rb_control_frame_t *reg_cf int argc, VALUE *argv, VALUE *stack_bottom) { RB_DEBUG_COUNTER_INC(ccf_cfunc_with_frame); - const struct rb_callinfo *ci = calling->ci; + const struct rb_callinfo *ci = calling->cd->ci; const struct rb_callcache *cc = calling->cc; VALUE val; const rb_callable_method_entry_t *me = vm_cc_cme(cc); @@ -3489,7 +3491,7 @@ vm_call_cfunc_with_frame(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp static VALUE vm_call_cfunc_other(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling) { - const struct rb_callinfo *ci = calling->ci; + const struct rb_callinfo *ci = calling->cd->ci; RB_DEBUG_COUNTER_INC(ccf_cfunc_other); CALLER_SETUP_ARG(reg_cfp, calling, ci, ALLOW_HEAP_ARGV_KEEP_KWSPLAT); @@ -3574,7 +3576,7 @@ vm_call_cfunc_only_splat_kw(rb_execution_context_t *ec, rb_control_frame_t *reg_ static VALUE vm_call_cfunc(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling) { - const struct rb_callinfo *ci = calling->ci; + const struct rb_callinfo *ci = calling->cd->ci; RB_DEBUG_COUNTER_INC(ccf_cfunc); if (IS_ARGS_SPLAT(ci)) { @@ -3693,11 +3695,11 @@ vm_call_iseq_bmethod(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct const int arg_size = ISEQ_BODY(iseq)->param.size; int opt_pc; - if (vm_ci_flag(calling->ci) & VM_CALL_ARGS_SIMPLE) { - opt_pc = vm_callee_setup_block_arg(ec, calling, calling->ci, iseq, argv, arg_setup_method); + if (vm_ci_flag(calling->cd->ci) & VM_CALL_ARGS_SIMPLE) { + opt_pc = vm_callee_setup_block_arg(ec, calling, calling->cd->ci, iseq, argv, arg_setup_method); } else { - opt_pc = setup_parameters_complex(ec, iseq, calling, calling->ci, argv, arg_setup_method); + opt_pc = setup_parameters_complex(ec, iseq, calling, calling->cd->ci, argv, arg_setup_method); } cfp->sp = argv - 1; // -1 for the receiver @@ -3722,7 +3724,7 @@ vm_call_noniseq_bmethod(rb_execution_context_t *ec, rb_control_frame_t *cfp, str VALUE *argv; int argc; - CALLER_SETUP_ARG(cfp, calling, calling->ci, ALLOW_HEAP_ARGV); + CALLER_SETUP_ARG(cfp, calling, calling->cd->ci, ALLOW_HEAP_ARGV); if (UNLIKELY(calling->heap_argv)) { argv = RARRAY_PTR(calling->heap_argv); cfp->sp -= 2; @@ -3908,7 +3910,10 @@ vm_call_symbol(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, } } - calling->ci = &VM_CI_ON_STACK(mid, flags, argc, vm_ci_kwarg(ci)); + calling->cd = &(struct rb_call_data) { + .ci = &VM_CI_ON_STACK(mid, flags, argc, vm_ci_kwarg(ci)), + .cc = NULL, + }; calling->cc = &VM_CC_ON_STACK(klass, vm_call_general, { .method_missing_reason = missing_reason }, @@ -3943,7 +3948,7 @@ vm_call_symbol(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, static VALUE vm_call_opt_send0(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling, int flags) { - const struct rb_callinfo *ci = calling->ci; + const struct rb_callinfo *ci = calling->cd->ci; int i; VALUE sym; @@ -3981,7 +3986,7 @@ static VALUE vm_call_opt_send_complex(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling) { RB_DEBUG_COUNTER_INC(ccf_opt_send_complex); - const struct rb_callinfo *ci = calling->ci; + const struct rb_callinfo *ci = calling->cd->ci; int flags = VM_CALL_FCALL; VALUE sym; @@ -4006,7 +4011,7 @@ static VALUE vm_call_opt_send_simple(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling) { RB_DEBUG_COUNTER_INC(ccf_opt_send_simple); - return vm_call_opt_send0(ec, reg_cfp, calling, vm_ci_flag(calling->ci) | VM_CALL_FCALL); + return vm_call_opt_send0(ec, reg_cfp, calling, vm_ci_flag(calling->cd->ci) | VM_CALL_FCALL); } static VALUE @@ -4014,7 +4019,7 @@ vm_call_opt_send(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct { RB_DEBUG_COUNTER_INC(ccf_opt_send); - const struct rb_callinfo *ci = calling->ci; + const struct rb_callinfo *ci = calling->cd->ci; int flags = vm_ci_flag(ci); if (UNLIKELY(!(flags & VM_CALL_ARGS_SIMPLE) && @@ -4051,7 +4056,10 @@ vm_call_method_missing_body(rb_execution_context_t *ec, rb_control_frame_t *reg_ INC_SP(1); ec->method_missing_reason = reason; - calling->ci = &VM_CI_ON_STACK(idMethodMissing, flag, argc, vm_ci_kwarg(orig_ci)); + calling->cd = &(struct rb_call_data) { + .ci = &VM_CI_ON_STACK(idMethodMissing, flag, argc, vm_ci_kwarg(orig_ci)), + .cc = NULL, + }; calling->cc = &VM_CC_ON_STACK(Qundef, vm_call_general, {{ 0 }}, rb_callable_method_entry_without_refinements(CLASS_OF(calling->recv), idMethodMissing, NULL)); return vm_call_method(ec, reg_cfp, calling); @@ -4060,7 +4068,7 @@ vm_call_method_missing_body(rb_execution_context_t *ec, rb_control_frame_t *reg_ static VALUE vm_call_method_missing(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling) { - return vm_call_method_missing_body(ec, reg_cfp, calling, calling->ci, vm_cc_cmethod_missing_reason(calling->cc)); + return vm_call_method_missing_body(ec, reg_cfp, calling, calling->cd->ci, vm_cc_cmethod_missing_reason(calling->cc)); } static const rb_callable_method_entry_t *refined_method_callable_without_refinement(const rb_callable_method_entry_t *me); @@ -4069,7 +4077,7 @@ vm_call_zsuper(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_ca { klass = RCLASS_SUPER(klass); - const rb_callable_method_entry_t *cme = klass ? rb_callable_method_entry(klass, vm_ci_mid(calling->ci)) : NULL; + const rb_callable_method_entry_t *cme = klass ? rb_callable_method_entry(klass, vm_ci_mid(calling->cd->ci)) : NULL; if (cme == NULL) { return vm_call_method_nome(ec, cfp, calling); } @@ -4138,7 +4146,7 @@ refined_method_callable_without_refinement(const rb_callable_method_entry_t *me) static const rb_callable_method_entry_t * search_refined_method(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling) { - ID mid = vm_ci_mid(calling->ci); + ID mid = vm_ci_mid(calling->cd->ci); const rb_cref_t *cref = vm_get_cref(cfp->ep); const struct rb_callcache * const cc = calling->cc; const rb_callable_method_entry_t *cme = vm_cc_cme(cc); @@ -4221,7 +4229,7 @@ vm_call_opt_call(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct { RB_DEBUG_COUNTER_INC(ccf_opt_call); - const struct rb_callinfo *ci = calling->ci; + const struct rb_callinfo *ci = calling->cd->ci; VALUE procval = calling->recv; return vm_invoke_block_opt_call(ec, reg_cfp, calling, ci, VM_BH_FROM_PROC(procval)); } @@ -4232,7 +4240,7 @@ vm_call_opt_block_call(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, RB_DEBUG_COUNTER_INC(ccf_opt_block_call); VALUE block_handler = VM_ENV_BLOCK_HANDLER(VM_CF_LEP(reg_cfp)); - const struct rb_callinfo *ci = calling->ci; + const struct rb_callinfo *ci = calling->cd->ci; if (BASIC_OP_UNREDEFINED_P(BOP_CALL, PROC_REDEFINED_OP_FLAG)) { return vm_invoke_block_opt_call(ec, reg_cfp, calling, ci, block_handler); @@ -4343,7 +4351,7 @@ vm_call_optimized(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb static VALUE vm_call_method_each_type(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling) { - const struct rb_callinfo *ci = calling->ci; + const struct rb_callinfo *ci = calling->cd->ci; const struct rb_callcache *cc = calling->cc; const rb_callable_method_entry_t *cme = vm_cc_cme(cc); VALUE v; @@ -4440,7 +4448,7 @@ static VALUE vm_call_method_nome(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling) { /* method missing */ - const struct rb_callinfo *ci = calling->ci; + const struct rb_callinfo *ci = calling->cd->ci; const int stat = ci_missing_reason(ci); if (vm_ci_mid(ci) == idMethodMissing) { @@ -4474,7 +4482,7 @@ vm_defined_class_for_protected_call(const rb_callable_method_entry_t *me) static inline VALUE vm_call_method(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling) { - const struct rb_callinfo *ci = calling->ci; + const struct rb_callinfo *ci = calling->cd->ci; const struct rb_callcache *cc = calling->cc; VM_ASSERT(callable_method_entry_p(vm_cc_cme(cc))); @@ -5473,7 +5481,7 @@ vm_invokeblock_i(struct rb_execution_context_struct *ec, struct rb_control_frame_struct *reg_cfp, struct rb_calling_info *calling) { - const struct rb_callinfo *ci = calling->ci; + const struct rb_callinfo *ci = calling->cd->ci; VALUE block_handler = VM_CF_BLOCK_HANDLER(GET_CFP()); if (block_handler == VM_BLOCK_HANDLER_NONE) { @@ -5508,7 +5516,7 @@ vm_sendish( .kw_splat = IS_ARGS_KW_SPLAT(ci) > 0, .recv = recv, .argc = argc, - .ci = ci, + .cd = cd, }; switch (method_explorer) { @@ -5518,7 +5526,6 @@ vm_sendish( break; case mexp_search_super: calling.cc = cc = vm_search_super_method(reg_cfp, cd, recv); - calling.ci = cd->ci; // TODO: does it safe? val = vm_cc_call(cc)(ec, GET_CFP(), &calling); break; case mexp_search_invokeblock: diff --git a/vm_method.c b/vm_method.c index d925e4f2fb..221f184267 100644 --- a/vm_method.c +++ b/vm_method.c @@ -322,6 +322,7 @@ invalidate_all_cc(void *vstart, void *vend, size_t stride, void *data) RCLASS_CC_TBL(v) = NULL; } } + if (ptr) { asan_poison_object(v); }