`Primitive.mandatory_only?` for fast path

Compare with the C methods, A built-in methods written in Ruby is
slower if only mandatory parameters are given because it needs to
check the argumens and fill default values for optional and keyword
parameters (C methods can check the number of parameters with `argc`,
so there are no overhead). Passing mandatory arguments are common
(optional arguments are exceptional, in many cases) so it is important
to provide the fast path for such common cases.

`Primitive.mandatory_only?` is a special builtin function used with
`if` expression like that:

```ruby
  def self.at(time, subsec = false, unit = :microsecond, in: nil)
    if Primitive.mandatory_only?
      Primitive.time_s_at1(time)
    else
      Primitive.time_s_at(time, subsec, unit, Primitive.arg!(:in))
    end
  end
```

and it makes two ISeq,

```
  def self.at(time, subsec = false, unit = :microsecond, in: nil)
    Primitive.time_s_at(time, subsec, unit, Primitive.arg!(:in))
  end

  def self.at(time)
    Primitive.time_s_at1(time)
  end
```

and (2) is pointed by (1). Note that `Primitive.mandatory_only?`
should be used only in a condition of an `if` statement and the
`if` statement should be equal to the methdo body (you can not
put any expression before and after the `if` statement).

A method entry with `mandatory_only?` (`Time.at` on the above case)
is marked as `iseq_overload`. When the method will be dispatch only
with mandatory arguments (`Time.at(0)` for example), make another
method entry with ISeq (2) as mandatory only method entry and it
will be cached in an inline method cache.

The idea is similar discussed in https://bugs.ruby-lang.org/issues/16254
but it only checks mandatory parameters or more, because many cases
only mandatory parameters are given. If we find other cases (optional
or keyword parameters are used frequently and it hurts performance),
we can extend the feature.
This commit is contained in:
Koichi Sasada 2021-11-13 02:12:20 +09:00
Родитель f9638c3b17
Коммит b1b73936c1
11 изменённых файлов: 177 добавлений и 16 удалений

Просмотреть файл

@ -791,8 +791,10 @@ rb_iseq_compile_node(rb_iseq_t *iseq, const NODE *node)
}
case ISEQ_TYPE_METHOD:
{
ISEQ_COMPILE_DATA(iseq)->root_node = node->nd_body;
ADD_TRACE(ret, RUBY_EVENT_CALL);
CHECK(COMPILE(ret, "scoped node", node->nd_body));
ISEQ_COMPILE_DATA(iseq)->root_node = node->nd_body;
ADD_TRACE(ret, RUBY_EVENT_RETURN);
ISEQ_COMPILE_DATA(iseq)->last_line = nd_line(node);
break;
@ -7884,6 +7886,65 @@ compile_builtin_arg(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *node, c
UNKNOWN_NODE("arg!", node, COMPILE_NG);
}
static NODE *
mandatory_node(const rb_iseq_t *iseq, const NODE *cond_node)
{
const NODE *node = ISEQ_COMPILE_DATA(iseq)->root_node;
if (nd_type(node) == NODE_IF && node->nd_cond == cond_node) {
return node->nd_body;
}
else {
rb_bug("mandatory_node: can't find mandatory node");
}
}
static int
compile_builtin_mandatory_only_method(rb_iseq_t *iseq, const NODE *node, const NODE *line_node)
{
// argumens
struct rb_args_info args = {
.pre_args_num = iseq->body->param.lead_num,
};
NODE args_node;
rb_node_init(&args_node, NODE_ARGS, 0, 0, (VALUE)&args);
// local table without non-mandatory parameters
const int skip_local_size = iseq->body->param.size - iseq->body->param.lead_num;
const int table_size = iseq->body->local_table_size - skip_local_size;
ID *tbl = ALLOCA_N(ID, table_size + 1);
tbl[0] = table_size;
int i;
// lead parameters
for (i=0; i<iseq->body->param.lead_num; i++) {
tbl[i+1] = iseq->body->local_table[i];
}
// local variables
for (; i<table_size; i++) {
tbl[i+1] = iseq->body->local_table[i + skip_local_size];
}
NODE scope_node;
rb_node_init(&scope_node, NODE_SCOPE, (VALUE)tbl, (VALUE)mandatory_node(iseq, node), (VALUE)&args_node);
rb_ast_body_t ast = {
.root = &scope_node,
.compile_option = 0,
.script_lines = iseq->body->variable.script_lines,
};
int prev_inline_index = GET_VM()->builtin_inline_index;
iseq->body->mandatory_only_iseq =
rb_iseq_new_with_opt(&ast, rb_iseq_base_label(iseq),
rb_iseq_path(iseq), rb_iseq_realpath(iseq),
INT2FIX(nd_line(line_node)), NULL, 0,
ISEQ_TYPE_METHOD, ISEQ_COMPILE_DATA(iseq)->option);
GET_VM()->builtin_inline_index = prev_inline_index;
return COMPILE_OK;
}
static int
compile_builtin_function_call(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, const NODE *line_node, int popped,
const rb_iseq_t *parent_block, LINK_ANCHOR *args, const char *builtin_func)
@ -7922,6 +7983,18 @@ compile_builtin_function_call(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NOD
else if (strcmp("arg!", builtin_func) == 0) {
return compile_builtin_arg(iseq, ret, args_node, line_node, popped);
}
else if (strcmp("mandatory_only?", builtin_func) == 0) {
if (popped) {
rb_bug("mandatory_only? should be in if condition");
}
else if (!LIST_INSN_SIZE_ZERO(ret)) {
rb_bug("mandatory_only? should be put on top");
}
ADD_INSN1(ret, line_node, putobject, Qfalse);
return compile_builtin_mandatory_only_method(iseq, node, line_node);
return COMPILE_OK;
}
else if (1) {
rb_bug("can't find builtin function:%s", builtin_func);
}
@ -11628,6 +11701,7 @@ ibf_dump_iseq_each(struct ibf_dump *dump, const rb_iseq_t *iseq)
const ibf_offset_t catch_table_offset = ibf_dump_catch_table(dump, iseq);
const int parent_iseq_index = ibf_dump_iseq(dump, iseq->body->parent_iseq);
const int local_iseq_index = ibf_dump_iseq(dump, iseq->body->local_iseq);
const int mandatory_only_iseq_index = ibf_dump_iseq(dump, iseq->body->mandatory_only_iseq);
const ibf_offset_t ci_entries_offset = ibf_dump_ci_entries(dump, iseq);
const ibf_offset_t outer_variables_offset = ibf_dump_outer_variables(dump, iseq);
@ -11690,6 +11764,7 @@ ibf_dump_iseq_each(struct ibf_dump *dump, const rb_iseq_t *iseq)
ibf_dump_write_small_value(dump, IBF_BODY_OFFSET(catch_table_offset));
ibf_dump_write_small_value(dump, parent_iseq_index);
ibf_dump_write_small_value(dump, local_iseq_index);
ibf_dump_write_small_value(dump, mandatory_only_iseq_index);
ibf_dump_write_small_value(dump, IBF_BODY_OFFSET(ci_entries_offset));
ibf_dump_write_small_value(dump, IBF_BODY_OFFSET(outer_variables_offset));
ibf_dump_write_small_value(dump, body->variable.flip_count);
@ -11797,6 +11872,7 @@ ibf_load_iseq_each(struct ibf_load *load, rb_iseq_t *iseq, ibf_offset_t offset)
const ibf_offset_t catch_table_offset = (ibf_offset_t)IBF_BODY_OFFSET(ibf_load_small_value(load, &reading_pos));
const int parent_iseq_index = (int)ibf_load_small_value(load, &reading_pos);
const int local_iseq_index = (int)ibf_load_small_value(load, &reading_pos);
const int mandatory_only_iseq_index = (int)ibf_load_small_value(load, &reading_pos);
const ibf_offset_t ci_entries_offset = (ibf_offset_t)IBF_BODY_OFFSET(ibf_load_small_value(load, &reading_pos));
const ibf_offset_t outer_variables_offset = (ibf_offset_t)IBF_BODY_OFFSET(ibf_load_small_value(load, &reading_pos));
const rb_snum_t variable_flip_count = (rb_snum_t)ibf_load_small_value(load, &reading_pos);
@ -11859,6 +11935,7 @@ ibf_load_iseq_each(struct ibf_load *load, rb_iseq_t *iseq, ibf_offset_t offset)
load_body->catch_table = ibf_load_catch_table(load, catch_table_offset, catch_table_size);
load_body->parent_iseq = ibf_load_iseq(load, (const rb_iseq_t *)(VALUE)parent_iseq_index);
load_body->local_iseq = ibf_load_iseq(load, (const rb_iseq_t *)(VALUE)local_iseq_index);
load_body->mandatory_only_iseq = ibf_load_iseq(load, (const rb_iseq_t *)(VALUE)mandatory_only_iseq_index);
ibf_load_code(load, iseq, bytecode_offset, bytecode_size, iseq_size);
#if VM_INSN_INFO_TABLE_IMPL == 2

14
gc.c
Просмотреть файл

@ -2960,7 +2960,7 @@ cc_table_mark_i(ID id, VALUE ccs_ptr, void *data_ptr)
for (int i=0; i<ccs->len; i++) {
VM_ASSERT(data->klass == ccs->entries[i].cc->klass);
VM_ASSERT(ccs->cme == vm_cc_cme(ccs->entries[i].cc));
VM_ASSERT(vm_cc_check_cme(ccs->entries[i].cc, ccs->cme));
gc_mark(data->objspace, (VALUE)ccs->entries[i].ci);
gc_mark(data->objspace, (VALUE)ccs->entries[i].cc);
@ -6443,10 +6443,11 @@ mark_method_entry(rb_objspace_t *objspace, const rb_method_entry_t *me)
if (def) {
switch (def->type) {
case VM_METHOD_TYPE_ISEQ:
if (def->body.iseq.iseqptr) gc_mark(objspace, (VALUE)def->body.iseq.iseqptr);
gc_mark(objspace, (VALUE)def->body.iseq.cref);
break;
case VM_METHOD_TYPE_ISEQ:
if (def->body.iseq.iseqptr) gc_mark(objspace, (VALUE)def->body.iseq.iseqptr);
gc_mark(objspace, (VALUE)def->body.iseq.cref);
if (def->body.iseq.mandatory_only_cme) gc_mark(objspace, (VALUE)def->body.iseq.mandatory_only_cme);
break;
case VM_METHOD_TYPE_ATTRSET:
case VM_METHOD_TYPE_IVAR:
gc_mark(objspace, def->body.attr.location);
@ -9612,6 +9613,9 @@ gc_ref_update_method_entry(rb_objspace_t *objspace, rb_method_entry_t *me)
TYPED_UPDATE_IF_MOVED(objspace, rb_iseq_t *, def->body.iseq.iseqptr);
}
TYPED_UPDATE_IF_MOVED(objspace, rb_cref_t *, def->body.iseq.cref);
if (def->body.iseq.mandatory_only_cme) {
TYPED_UPDATE_IF_MOVED(objspace, rb_callable_method_entry_t *, def->body.iseq.mandatory_only_cme);
}
break;
case VM_METHOD_TYPE_ATTRSET:
case VM_METHOD_TYPE_IVAR:

4
iseq.c
Просмотреть файл

@ -272,6 +272,9 @@ rb_iseq_update_references(rb_iseq_t *iseq)
if (body->parent_iseq) {
body->parent_iseq = (struct rb_iseq_struct *)rb_gc_location((VALUE)body->parent_iseq);
}
if (body->mandatory_only_iseq) {
body->mandatory_only_iseq = (struct rb_iseq_struct *)rb_gc_location((VALUE)body->mandatory_only_iseq);
}
if (body->call_data) {
for (unsigned int i=0; i<body->ci_size; i++) {
struct rb_call_data *cds = body->call_data;
@ -351,6 +354,7 @@ rb_iseq_mark(const rb_iseq_t *iseq)
rb_gc_mark_movable(body->location.label);
rb_gc_mark_movable(body->location.base_label);
rb_gc_mark_movable(body->location.pathobj);
RUBY_MARK_MOVABLE_UNLESS_NULL((VALUE)body->mandatory_only_iseq);
RUBY_MARK_MOVABLE_UNLESS_NULL((VALUE)body->parent_iseq);
if (body->call_data) {

1
iseq.h
Просмотреть файл

@ -117,6 +117,7 @@ struct iseq_compile_data {
const rb_compile_option_t *option;
struct rb_id_table *ivar_cache_table;
const struct rb_builtin_function *builtin_function_table;
const NODE *root_node;
#if OPT_SUPPORT_JOKE
st_table *labels_table;
#endif

Просмотреть файл

@ -132,8 +132,9 @@ typedef struct rb_iseq_struct rb_iseq_t;
#endif
typedef struct rb_method_iseq_struct {
rb_iseq_t * iseqptr; /*!< iseq pointer, should be separated from iseqval */
const rb_iseq_t * iseqptr; /*!< iseq pointer, should be separated from iseqval */
rb_cref_t * cref; /*!< class reference, should be marked */
const rb_callable_method_entry_t *mandatory_only_cme;
} rb_method_iseq_t; /* check rb_add_method_iseq() when modify the fields */
typedef struct rb_method_cfunc_struct {
@ -171,7 +172,8 @@ enum method_optimized_type {
struct rb_method_definition_struct {
BITFIELD(rb_method_type_t, type, VM_METHOD_TYPE_MINIMUM_BITS);
int alias_count : 28;
unsigned int iseq_overload: 1;
int alias_count : 27;
int complemented_count : 28;
union {

Просмотреть файл

@ -100,6 +100,7 @@ def collect_builtin base, tree, name, bs, inlines, locals = nil
when :call, :command_call # CALL
_, recv, sep, mid, (_, args) = tree
end
if mid
raise "unknown sexp: #{mid.inspect}" unless %i[@ident @const].include?(mid.first)
_, mid, (lineno,) = mid
@ -126,7 +127,7 @@ def collect_builtin base, tree, name, bs, inlines, locals = nil
args.pop unless (args ||= []).last
argc = args.size
if /(.+)\!\z/ =~ func_name
if /(.+)[\!\?]\z/ =~ func_name
case $1
when 'attr'
text = inline_text(argc, args.first)
@ -156,6 +157,8 @@ def collect_builtin base, tree, name, bs, inlines, locals = nil
func_name = nil # required
inlines[inlines.size] = [lineno, text, nil, nil]
argc -= 1
when 'mandatory_only'
func_name = nil
when 'arg'
argc == 1 or raise "unexpected argument number #{argc}"
(arg = args.first)[0] == :symbol_literal or raise "symbol literal expected #{args}"

Просмотреть файл

@ -446,6 +446,25 @@ vm_ccs_p(const struct rb_class_cc_entries *ccs)
{
return ccs->debug_sig == ~(VALUE)ccs;
}
static inline bool
vm_cc_check_cme(const struct rb_callcache *cc, const rb_callable_method_entry_t *cme)
{
if (vm_cc_cme(cc) == cme ||
(cme->def->iseq_overload && vm_cc_cme(cc) == cme->def->body.iseq.mandatory_only_cme)) {
return true;
}
else {
#if 1
fprintf(stderr, "iseq_overload:%d mandatory_only_cme:%p eq:%d\n",
(int)cme->def->iseq_overload,
(void *)cme->def->body.iseq.mandatory_only_cme,
vm_cc_cme(cc) == cme->def->body.iseq.mandatory_only_cme);
#endif
return false;
}
}
#endif
// gc.c

Просмотреть файл

@ -482,6 +482,8 @@ struct rb_iseq_constant_body {
bool builtin_inline_p;
struct rb_id_table *outer_variables;
const rb_iseq_t *mandatory_only_iseq;
#if USE_MJIT
/* The following fields are MJIT related info. */
VALUE (*jit_func)(struct rb_execution_context_struct *,

Просмотреть файл

@ -455,7 +455,7 @@ gccct_method_search(rb_execution_context_t *ec, VALUE recv, ID mid, int argc)
if (LIKELY(!METHOD_ENTRY_INVALIDATED(cme) &&
cme->called_id == mid)) {
VM_ASSERT(vm_cc_cme(cc) == rb_callable_method_entry(klass, mid));
VM_ASSERT(vm_cc_check_cme(cc, rb_callable_method_entry(klass, mid)));
RB_DEBUG_COUNTER_INC(gccct_hit);
return cc;

Просмотреть файл

@ -678,7 +678,7 @@ rb_vm_frame_method_entry(const rb_control_frame_t *cfp)
return check_method_entry(ep[VM_ENV_DATA_INDEX_ME_CREF], TRUE);
}
static rb_iseq_t *
static const rb_iseq_t *
method_entry_iseqptr(const rb_callable_method_entry_t *me)
{
switch (me->def->type) {
@ -1754,13 +1754,16 @@ vm_ccs_verify(struct rb_class_cc_entries *ccs, ID mid, VALUE klass)
VM_ASSERT(vm_ci_mid(ci) == mid);
VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
VM_ASSERT(vm_cc_class_check(cc, klass));
VM_ASSERT(vm_cc_cme(cc) == ccs->cme);
VM_ASSERT(vm_cc_check_cme(cc, ccs->cme));
}
return TRUE;
}
#endif
#ifndef MJIT_HEADER
static const rb_callable_method_entry_t *overloaded_cme(const rb_callable_method_entry_t *cme);
static const struct rb_callcache *
vm_search_cc(const VALUE klass, const struct rb_callinfo * const ci)
{
@ -1829,7 +1832,6 @@ vm_search_cc(const VALUE klass, const struct rb_callinfo * const ci)
VM_ASSERT(cme == rb_callable_method_entry(klass, mid));
const struct rb_callcache *cc = vm_cc_new(klass, cme, vm_call_general);
METHOD_ENTRY_CACHED_SET((struct rb_callable_method_entry_struct *)cme);
if (ccs == NULL) {
@ -1846,6 +1848,14 @@ vm_search_cc(const VALUE klass, const struct rb_callinfo * const ci)
}
}
if ((cme->def->iseq_overload &&
(int)vm_ci_argc(ci) == method_entry_iseqptr(cme)->body->param.lead_num)) {
// use alternative
cme = overloaded_cme(cme);
METHOD_ENTRY_CACHED_SET((struct rb_callable_method_entry_struct *)cme);
// rp(cme);
}
const struct rb_callcache *cc = vm_cc_new(klass, cme, vm_call_general);
vm_ccs_push(klass, ccs, ci, cc);
VM_ASSERT(vm_cc_cme(cc) != NULL);
@ -3529,9 +3539,10 @@ vm_call_method_each_type(rb_execution_context_t *ec, rb_control_frame_t *cfp, st
{
const struct rb_callinfo *ci = calling->ci;
const struct rb_callcache *cc = calling->cc;
const rb_callable_method_entry_t *cme = vm_cc_cme(cc);
VALUE v;
switch (vm_cc_cme(cc)->def->type) {
switch (cme->def->type) {
case VM_METHOD_TYPE_ISEQ:
CC_SET_FASTPATH(cc, vm_call_iseq_setup, TRUE);
return vm_call_iseq_setup(ec, cfp, calling);

Просмотреть файл

@ -208,6 +208,10 @@ clear_method_cache_by_id_in_class(VALUE klass, ID mid)
vm_cme_invalidate((rb_callable_method_entry_t *)cme);
RB_DEBUG_COUNTER_INC(cc_invalidate_tree_cme);
if (cme->def->iseq_overload) {
vm_cme_invalidate((rb_callable_method_entry_t *)cme->def->body.iseq.mandatory_only_cme);
}
}
// invalidate complement tbl
@ -451,10 +455,13 @@ rb_method_definition_set(const rb_method_entry_t *me, rb_method_definition_t *de
case VM_METHOD_TYPE_ISEQ:
{
rb_method_iseq_t *iseq_body = (rb_method_iseq_t *)opts;
const rb_iseq_t *iseq = iseq_body->iseqptr;
rb_cref_t *method_cref, *cref = iseq_body->cref;
/* setup iseq first (before invoking GC) */
RB_OBJ_WRITE(me, &def->body.iseq.iseqptr, iseq_body->iseqptr);
RB_OBJ_WRITE(me, &def->body.iseq.iseqptr, iseq);
if (iseq->body->mandatory_only_iseq) def->iseq_overload = 1;
if (0) vm_cref_dump("rb_method_definition_create", cref);
@ -531,7 +538,8 @@ method_definition_reset(const rb_method_entry_t *me)
case VM_METHOD_TYPE_ISEQ:
RB_OBJ_WRITTEN(me, Qundef, def->body.iseq.iseqptr);
RB_OBJ_WRITTEN(me, Qundef, def->body.iseq.cref);
break;
RB_OBJ_WRITTEN(me, Qundef, def->body.iseq.mandatory_only_cme);
break;
case VM_METHOD_TYPE_ATTRSET:
case VM_METHOD_TYPE_IVAR:
RB_OBJ_WRITTEN(me, Qundef, def->body.attr.location);
@ -893,6 +901,35 @@ rb_method_entry_make(VALUE klass, ID mid, VALUE defined_class, rb_method_visibil
return me;
}
static const rb_callable_method_entry_t *
overloaded_cme(const rb_callable_method_entry_t *cme)
{
VM_ASSERT(cme->def->iseq_overload);
VM_ASSERT(cme->def->type == VM_METHOD_TYPE_ISEQ);
VM_ASSERT(cme->def->body.iseq.iseqptr != NULL);
const rb_callable_method_entry_t *monly_cme = cme->def->body.iseq.mandatory_only_cme;
if (monly_cme && !METHOD_ENTRY_INVALIDATED(monly_cme)) {
// ok
}
else {
rb_method_definition_t *def = rb_method_definition_create(VM_METHOD_TYPE_ISEQ, cme->def->original_id);
def->body.iseq.cref = cme->def->body.iseq.cref;
def->body.iseq.iseqptr = cme->def->body.iseq.iseqptr->body->mandatory_only_iseq;
rb_method_entry_t *me = rb_method_entry_alloc(cme->called_id,
cme->owner,
cme->defined_class,
def);
METHOD_ENTRY_VISI_SET(me, METHOD_ENTRY_VISI(cme));
RB_OBJ_WRITE(cme, &cme->def->body.iseq.mandatory_only_cme, me);
monly_cme = (rb_callable_method_entry_t *)me;
}
return monly_cme;
}
#define CALL_METHOD_HOOK(klass, hook, mid) do { \
const VALUE arg = ID2SYM(mid); \
VALUE recv_class = (klass); \
@ -932,6 +969,7 @@ rb_add_method_iseq(VALUE klass, ID mid, const rb_iseq_t *iseq, rb_cref_t *cref,
iseq_body.iseqptr = iseq;
iseq_body.cref = cref;
rb_add_method(klass, mid, VM_METHOD_TYPE_ISEQ, &iseq_body, visi);
}
@ -1876,7 +1914,7 @@ rb_method_definition_eq(const rb_method_definition_t *d1, const rb_method_defini
switch (d1->type) {
case VM_METHOD_TYPE_ISEQ:
return d1->body.iseq.iseqptr == d2->body.iseq.iseqptr;
return d1->body.iseq.iseqptr == d2->body.iseq.iseqptr;
case VM_METHOD_TYPE_CFUNC:
return
d1->body.cfunc.func == d2->body.cfunc.func &&