2020-04-10 08:11:40 +03:00
|
|
|
#ifndef RUBY_VM_CALLINFO_H /*-*-C-*-vi:se ft=c:*/
|
|
|
|
#define RUBY_VM_CALLINFO_H
|
|
|
|
/**
|
|
|
|
* @author Ruby developers <ruby-core@ruby-lang.org>
|
|
|
|
* @copyright This file is a part of the programming language Ruby.
|
|
|
|
* Permission is hereby granted, to either redistribute and/or
|
|
|
|
* modify this file, provided that the conditions mentioned in the
|
|
|
|
* file COPYING are met. Consult the file for details.
|
|
|
|
*/
|
|
|
|
|
VALUE size packed callinfo (ci).
Now, rb_call_info contains how to call the method with tuple of
(mid, orig_argc, flags, kwarg). Most of cases, kwarg == NULL and
mid+argc+flags only requires 64bits. So this patch packed
rb_call_info to VALUE (1 word) on such cases. If we can not
represent it in VALUE, then use imemo_callinfo which contains
conventional callinfo (rb_callinfo, renamed from rb_call_info).
iseq->body->ci_kw_size is removed because all of callinfo is VALUE
size (packed ci or a pointer to imemo_callinfo).
To access ci information, we need to use these functions:
vm_ci_mid(ci), _flag(ci), _argc(ci), _kwarg(ci).
struct rb_call_info_kw_arg is renamed to rb_callinfo_kwarg.
rb_funcallv_with_cc() and rb_method_basic_definition_p_with_cc()
is temporary removed because cd->ci should be marked.
2020-01-08 02:20:36 +03:00
|
|
|
#include "debug_counter.h"
|
2022-06-06 17:48:42 +03:00
|
|
|
#include "internal/class.h"
|
2022-10-03 18:14:32 +03:00
|
|
|
#include "shape.h"
|
VALUE size packed callinfo (ci).
Now, rb_call_info contains how to call the method with tuple of
(mid, orig_argc, flags, kwarg). Most of cases, kwarg == NULL and
mid+argc+flags only requires 64bits. So this patch packed
rb_call_info to VALUE (1 word) on such cases. If we can not
represent it in VALUE, then use imemo_callinfo which contains
conventional callinfo (rb_callinfo, renamed from rb_call_info).
iseq->body->ci_kw_size is removed because all of callinfo is VALUE
size (packed ci or a pointer to imemo_callinfo).
To access ci information, we need to use these functions:
vm_ci_mid(ci), _flag(ci), _argc(ci), _kwarg(ci).
struct rb_call_info_kw_arg is renamed to rb_callinfo_kwarg.
rb_funcallv_with_cc() and rb_method_basic_definition_p_with_cc()
is temporary removed because cd->ci should be marked.
2020-01-08 02:20:36 +03:00
|
|
|
|
|
|
|
enum vm_call_flag_bits {
|
2023-04-01 20:17:57 +03:00
|
|
|
VM_CALL_ARGS_SPLAT_bit, // m(*args)
|
|
|
|
VM_CALL_ARGS_BLOCKARG_bit, // m(&block)
|
|
|
|
VM_CALL_FCALL_bit, // m(args) # receiver is self
|
|
|
|
VM_CALL_VCALL_bit, // m # method call that looks like a local variable
|
2024-06-28 17:11:35 +03:00
|
|
|
VM_CALL_ARGS_SIMPLE_bit, // !(ci->flag & (SPLAT|BLOCKARG|KWARG|KW_SPLAT|FORWARDING)) && !has_block_iseq
|
2023-04-01 20:17:57 +03:00
|
|
|
VM_CALL_KWARG_bit, // has kwarg
|
|
|
|
VM_CALL_KW_SPLAT_bit, // m(**opts)
|
|
|
|
VM_CALL_TAILCALL_bit, // located at tail position
|
|
|
|
VM_CALL_SUPER_bit, // super
|
|
|
|
VM_CALL_ZSUPER_bit, // zsuper
|
|
|
|
VM_CALL_OPT_SEND_bit, // internal flag
|
|
|
|
VM_CALL_KW_SPLAT_MUT_bit, // kw splat hash can be modified (to avoid allocating a new one)
|
Add VM_CALL_ARGS_SPLAT_MUT callinfo flag
This flag is set when the caller has already created a new array to
handle a splat, such as for `f(*a, b)` and `f(*a, *b)`. Previously,
if `f` was defined as `def f(*a)`, these calls would create an extra
array on the callee side, instead of using the new array created
by the caller.
This modifies `setup_args_core` to set the flag whenver it would add
a `splatarray true` instruction. However, when `splatarray true` is
changed to `splatarray false` in the peephole optimizer, to avoid
unnecessary allocations on the caller side, the flag must be removed.
Add `optimize_args_splat_no_copy` and have the peephole optimizer call
that. This significantly simplifies the related peephole optimizer
code.
On the callee side, in `setup_parameters_complex`, set
`args->rest_dupped` to true if the flag is set.
This takes a similar approach for optimizing regular splats that was
previiously used for keyword splats in
d2c41b1bff1f3102544bb0d03d4e82356d034d33 (via VM_CALL_KW_SPLAT_MUT).
2023-11-23 21:47:24 +03:00
|
|
|
VM_CALL_ARGS_SPLAT_MUT_bit, // args splat can be modified (to avoid allocating a new one)
|
Optimized forwarding callers and callees
This patch optimizes forwarding callers and callees. It only optimizes methods that only take `...` as their parameter, and then pass `...` to other calls.
Calls it optimizes look like this:
```ruby
def bar(a) = a
def foo(...) = bar(...) # optimized
foo(123)
```
```ruby
def bar(a) = a
def foo(...) = bar(1, 2, ...) # optimized
foo(123)
```
```ruby
def bar(*a) = a
def foo(...)
list = [1, 2]
bar(*list, ...) # optimized
end
foo(123)
```
All variants of the above but using `super` are also optimized, including a bare super like this:
```ruby
def foo(...)
super
end
```
This patch eliminates intermediate allocations made when calling methods that accept `...`.
We can observe allocation elimination like this:
```ruby
def m
x = GC.stat(:total_allocated_objects)
yield
GC.stat(:total_allocated_objects) - x
end
def bar(a) = a
def foo(...) = bar(...)
def test
m { foo(123) }
end
test
p test # allocates 1 object on master, but 0 objects with this patch
```
```ruby
def bar(a, b:) = a + b
def foo(...) = bar(...)
def test
m { foo(1, b: 2) }
end
test
p test # allocates 2 objects on master, but 0 objects with this patch
```
How does it work?
-----------------
This patch works by using a dynamic stack size when passing forwarded parameters to callees.
The caller's info object (known as the "CI") contains the stack size of the
parameters, so we pass the CI object itself as a parameter to the callee.
When forwarding parameters, the forwarding ISeq uses the caller's CI to determine how much stack to copy, then copies the caller's stack before calling the callee.
The CI at the forwarded call site is adjusted using information from the caller's CI.
I think this description is kind of confusing, so let's walk through an example with code.
```ruby
def delegatee(a, b) = a + b
def delegator(...)
delegatee(...) # CI2 (FORWARDING)
end
def caller
delegator(1, 2) # CI1 (argc: 2)
end
```
Before we call the delegator method, the stack looks like this:
```
Executing Line | Code | Stack
---------------+---------------------------------------+--------
1| def delegatee(a, b) = a + b | self
2| | 1
3| def delegator(...) | 2
4| # |
5| delegatee(...) # CI2 (FORWARDING) |
6| end |
7| |
8| def caller |
-> 9| delegator(1, 2) # CI1 (argc: 2) |
10| end |
```
The ISeq for `delegator` is tagged as "forwardable", so when `caller` calls in
to `delegator`, it writes `CI1` on to the stack as a local variable for the
`delegator` method. The `delegator` method has a special local called `...`
that holds the caller's CI object.
Here is the ISeq disasm fo `delegator`:
```
== disasm: #<ISeq:delegator@-e:1 (1,0)-(1,39)>
local table (size: 1, argc: 0 [opts: 0, rest: -1, post: 0, block: -1, kw: -1@-1, kwrest: -1])
[ 1] "..."@0
0000 putself ( 1)[LiCa]
0001 getlocal_WC_0 "..."@0
0003 send <calldata!mid:delegatee, argc:0, FCALL|FORWARDING>, nil
0006 leave [Re]
```
The local called `...` will contain the caller's CI: CI1.
Here is the stack when we enter `delegator`:
```
Executing Line | Code | Stack
---------------+---------------------------------------+--------
1| def delegatee(a, b) = a + b | self
2| | 1
3| def delegator(...) | 2
-> 4| # | CI1 (argc: 2)
5| delegatee(...) # CI2 (FORWARDING) | cref_or_me
6| end | specval
7| | type
8| def caller |
9| delegator(1, 2) # CI1 (argc: 2) |
10| end |
```
The CI at `delegatee` on line 5 is tagged as "FORWARDING", so it knows to
memcopy the caller's stack before calling `delegatee`. In this case, it will
memcopy self, 1, and 2 to the stack before calling `delegatee`. It knows how much
memory to copy from the caller because `CI1` contains stack size information
(argc: 2).
Before executing the `send` instruction, we push `...` on the stack. The
`send` instruction pops `...`, and because it is tagged with `FORWARDING`, it
knows to memcopy (using the information in the CI it just popped):
```
== disasm: #<ISeq:delegator@-e:1 (1,0)-(1,39)>
local table (size: 1, argc: 0 [opts: 0, rest: -1, post: 0, block: -1, kw: -1@-1, kwrest: -1])
[ 1] "..."@0
0000 putself ( 1)[LiCa]
0001 getlocal_WC_0 "..."@0
0003 send <calldata!mid:delegatee, argc:0, FCALL|FORWARDING>, nil
0006 leave [Re]
```
Instruction 001 puts the caller's CI on the stack. `send` is tagged with
FORWARDING, so it reads the CI and _copies_ the callers stack to this stack:
```
Executing Line | Code | Stack
---------------+---------------------------------------+--------
1| def delegatee(a, b) = a + b | self
2| | 1
3| def delegator(...) | 2
4| # | CI1 (argc: 2)
-> 5| delegatee(...) # CI2 (FORWARDING) | cref_or_me
6| end | specval
7| | type
8| def caller | self
9| delegator(1, 2) # CI1 (argc: 2) | 1
10| end | 2
```
The "FORWARDING" call site combines information from CI1 with CI2 in order
to support passing other values in addition to the `...` value, as well as
perfectly forward splat args, kwargs, etc.
Since we're able to copy the stack from `caller` in to `delegator`'s stack, we
can avoid allocating objects.
I want to do this to eliminate object allocations for delegate methods.
My long term goal is to implement `Class#new` in Ruby and it uses `...`.
I was able to implement `Class#new` in Ruby
[here](https://github.com/ruby/ruby/pull/9289).
If we adopt the technique in this patch, then we can optimize allocating
objects that take keyword parameters for `initialize`.
For example, this code will allocate 2 objects: one for `SomeObject`, and one
for the kwargs:
```ruby
SomeObject.new(foo: 1)
```
If we combine this technique, plus implement `Class#new` in Ruby, then we can
reduce allocations for this common operation.
Co-Authored-By: John Hawthorn <john@hawthorn.email>
Co-Authored-By: Alan Wu <XrXr@users.noreply.github.com>
2024-04-15 20:48:53 +03:00
|
|
|
VM_CALL_FORWARDING_bit, // m(...)
|
VALUE size packed callinfo (ci).
Now, rb_call_info contains how to call the method with tuple of
(mid, orig_argc, flags, kwarg). Most of cases, kwarg == NULL and
mid+argc+flags only requires 64bits. So this patch packed
rb_call_info to VALUE (1 word) on such cases. If we can not
represent it in VALUE, then use imemo_callinfo which contains
conventional callinfo (rb_callinfo, renamed from rb_call_info).
iseq->body->ci_kw_size is removed because all of callinfo is VALUE
size (packed ci or a pointer to imemo_callinfo).
To access ci information, we need to use these functions:
vm_ci_mid(ci), _flag(ci), _argc(ci), _kwarg(ci).
struct rb_call_info_kw_arg is renamed to rb_callinfo_kwarg.
rb_funcallv_with_cc() and rb_method_basic_definition_p_with_cc()
is temporary removed because cd->ci should be marked.
2020-01-08 02:20:36 +03:00
|
|
|
VM_CALL__END
|
|
|
|
};
|
|
|
|
|
|
|
|
#define VM_CALL_ARGS_SPLAT (0x01 << VM_CALL_ARGS_SPLAT_bit)
|
|
|
|
#define VM_CALL_ARGS_BLOCKARG (0x01 << VM_CALL_ARGS_BLOCKARG_bit)
|
|
|
|
#define VM_CALL_FCALL (0x01 << VM_CALL_FCALL_bit)
|
|
|
|
#define VM_CALL_VCALL (0x01 << VM_CALL_VCALL_bit)
|
|
|
|
#define VM_CALL_ARGS_SIMPLE (0x01 << VM_CALL_ARGS_SIMPLE_bit)
|
|
|
|
#define VM_CALL_KWARG (0x01 << VM_CALL_KWARG_bit)
|
|
|
|
#define VM_CALL_KW_SPLAT (0x01 << VM_CALL_KW_SPLAT_bit)
|
|
|
|
#define VM_CALL_TAILCALL (0x01 << VM_CALL_TAILCALL_bit)
|
|
|
|
#define VM_CALL_SUPER (0x01 << VM_CALL_SUPER_bit)
|
|
|
|
#define VM_CALL_ZSUPER (0x01 << VM_CALL_ZSUPER_bit)
|
|
|
|
#define VM_CALL_OPT_SEND (0x01 << VM_CALL_OPT_SEND_bit)
|
Reduce allocations for keyword argument hashes
Previously, passing a keyword splat to a method always allocated
a hash on the caller side, and accepting arbitrary keywords in
a method allocated a separate hash on the callee side. Passing
explicit keywords to a method that accepted a keyword splat
did not allocate a hash on the caller side, but resulted in two
hashes allocated on the callee side.
This commit makes passing a single keyword splat to a method not
allocate a hash on the caller side. Passing multiple keyword
splats or a mix of explicit keywords and a keyword splat still
generates a hash on the caller side. On the callee side,
if arbitrary keywords are not accepted, it does not allocate a
hash. If arbitrary keywords are accepted, it will allocate a
hash, but this commit uses a callinfo flag to indicate whether
the caller already allocated a hash, and if so, the callee can
use the passed hash without duplicating it. So this commit
should make it so that a maximum of a single hash is allocated
during method calls.
To set the callinfo flag appropriately, method call argument
compilation checks if only a single keyword splat is given.
If only one keyword splat is given, the VM_CALL_KW_SPLAT_MUT
callinfo flag is not set, since in that case the keyword
splat is passed directly and not mutable. If more than one
splat is used, a new hash needs to be generated on the caller
side, and in that case the callinfo flag is set, indicating
the keyword splat is mutable by the callee.
In compile_hash, used for both hash and keyword argument
compilation, if compiling keyword arguments and only a
single keyword splat is used, pass the argument directly.
On the caller side, in vm_args.c, the callinfo flag needs to
be recognized and handled. Because the keyword splat
argument may not be a hash, it needs to be converted to a
hash first if not. Then, unless the callinfo flag is set,
the hash needs to be duplicated. The temporary copy of the
callinfo flag, kw_flag, is updated if a hash was duplicated,
to prevent the need to duplicate it again. If we are
converting to a hash or duplicating a hash, we need to update
the argument array, which can including duplicating the
positional splat array if one was passed. CALLER_SETUP_ARG
and a couple other places needs to be modified to handle
similar issues for other types of calls.
This includes fairly comprehensive tests for different ways
keywords are handled internally, checking that you get equal
results but that keyword splats on the caller side result in
distinct objects for keyword rest parameters.
Included are benchmarks for keyword argument calls.
Brief results when compiled without optimization:
def kw(a: 1) a end
def kws(**kw) kw end
h = {a: 1}
kw(a: 1) # about same
kw(**h) # 2.37x faster
kws(a: 1) # 1.30x faster
kws(**h) # 2.19x faster
kw(a: 1, **h) # 1.03x slower
kw(**h, **h) # about same
kws(a: 1, **h) # 1.16x faster
kws(**h, **h) # 1.14x faster
2020-02-24 23:05:07 +03:00
|
|
|
#define VM_CALL_KW_SPLAT_MUT (0x01 << VM_CALL_KW_SPLAT_MUT_bit)
|
Add VM_CALL_ARGS_SPLAT_MUT callinfo flag
This flag is set when the caller has already created a new array to
handle a splat, such as for `f(*a, b)` and `f(*a, *b)`. Previously,
if `f` was defined as `def f(*a)`, these calls would create an extra
array on the callee side, instead of using the new array created
by the caller.
This modifies `setup_args_core` to set the flag whenver it would add
a `splatarray true` instruction. However, when `splatarray true` is
changed to `splatarray false` in the peephole optimizer, to avoid
unnecessary allocations on the caller side, the flag must be removed.
Add `optimize_args_splat_no_copy` and have the peephole optimizer call
that. This significantly simplifies the related peephole optimizer
code.
On the callee side, in `setup_parameters_complex`, set
`args->rest_dupped` to true if the flag is set.
This takes a similar approach for optimizing regular splats that was
previiously used for keyword splats in
d2c41b1bff1f3102544bb0d03d4e82356d034d33 (via VM_CALL_KW_SPLAT_MUT).
2023-11-23 21:47:24 +03:00
|
|
|
#define VM_CALL_ARGS_SPLAT_MUT (0x01 << VM_CALL_ARGS_SPLAT_MUT_bit)
|
Optimized forwarding callers and callees
This patch optimizes forwarding callers and callees. It only optimizes methods that only take `...` as their parameter, and then pass `...` to other calls.
Calls it optimizes look like this:
```ruby
def bar(a) = a
def foo(...) = bar(...) # optimized
foo(123)
```
```ruby
def bar(a) = a
def foo(...) = bar(1, 2, ...) # optimized
foo(123)
```
```ruby
def bar(*a) = a
def foo(...)
list = [1, 2]
bar(*list, ...) # optimized
end
foo(123)
```
All variants of the above but using `super` are also optimized, including a bare super like this:
```ruby
def foo(...)
super
end
```
This patch eliminates intermediate allocations made when calling methods that accept `...`.
We can observe allocation elimination like this:
```ruby
def m
x = GC.stat(:total_allocated_objects)
yield
GC.stat(:total_allocated_objects) - x
end
def bar(a) = a
def foo(...) = bar(...)
def test
m { foo(123) }
end
test
p test # allocates 1 object on master, but 0 objects with this patch
```
```ruby
def bar(a, b:) = a + b
def foo(...) = bar(...)
def test
m { foo(1, b: 2) }
end
test
p test # allocates 2 objects on master, but 0 objects with this patch
```
How does it work?
-----------------
This patch works by using a dynamic stack size when passing forwarded parameters to callees.
The caller's info object (known as the "CI") contains the stack size of the
parameters, so we pass the CI object itself as a parameter to the callee.
When forwarding parameters, the forwarding ISeq uses the caller's CI to determine how much stack to copy, then copies the caller's stack before calling the callee.
The CI at the forwarded call site is adjusted using information from the caller's CI.
I think this description is kind of confusing, so let's walk through an example with code.
```ruby
def delegatee(a, b) = a + b
def delegator(...)
delegatee(...) # CI2 (FORWARDING)
end
def caller
delegator(1, 2) # CI1 (argc: 2)
end
```
Before we call the delegator method, the stack looks like this:
```
Executing Line | Code | Stack
---------------+---------------------------------------+--------
1| def delegatee(a, b) = a + b | self
2| | 1
3| def delegator(...) | 2
4| # |
5| delegatee(...) # CI2 (FORWARDING) |
6| end |
7| |
8| def caller |
-> 9| delegator(1, 2) # CI1 (argc: 2) |
10| end |
```
The ISeq for `delegator` is tagged as "forwardable", so when `caller` calls in
to `delegator`, it writes `CI1` on to the stack as a local variable for the
`delegator` method. The `delegator` method has a special local called `...`
that holds the caller's CI object.
Here is the ISeq disasm fo `delegator`:
```
== disasm: #<ISeq:delegator@-e:1 (1,0)-(1,39)>
local table (size: 1, argc: 0 [opts: 0, rest: -1, post: 0, block: -1, kw: -1@-1, kwrest: -1])
[ 1] "..."@0
0000 putself ( 1)[LiCa]
0001 getlocal_WC_0 "..."@0
0003 send <calldata!mid:delegatee, argc:0, FCALL|FORWARDING>, nil
0006 leave [Re]
```
The local called `...` will contain the caller's CI: CI1.
Here is the stack when we enter `delegator`:
```
Executing Line | Code | Stack
---------------+---------------------------------------+--------
1| def delegatee(a, b) = a + b | self
2| | 1
3| def delegator(...) | 2
-> 4| # | CI1 (argc: 2)
5| delegatee(...) # CI2 (FORWARDING) | cref_or_me
6| end | specval
7| | type
8| def caller |
9| delegator(1, 2) # CI1 (argc: 2) |
10| end |
```
The CI at `delegatee` on line 5 is tagged as "FORWARDING", so it knows to
memcopy the caller's stack before calling `delegatee`. In this case, it will
memcopy self, 1, and 2 to the stack before calling `delegatee`. It knows how much
memory to copy from the caller because `CI1` contains stack size information
(argc: 2).
Before executing the `send` instruction, we push `...` on the stack. The
`send` instruction pops `...`, and because it is tagged with `FORWARDING`, it
knows to memcopy (using the information in the CI it just popped):
```
== disasm: #<ISeq:delegator@-e:1 (1,0)-(1,39)>
local table (size: 1, argc: 0 [opts: 0, rest: -1, post: 0, block: -1, kw: -1@-1, kwrest: -1])
[ 1] "..."@0
0000 putself ( 1)[LiCa]
0001 getlocal_WC_0 "..."@0
0003 send <calldata!mid:delegatee, argc:0, FCALL|FORWARDING>, nil
0006 leave [Re]
```
Instruction 001 puts the caller's CI on the stack. `send` is tagged with
FORWARDING, so it reads the CI and _copies_ the callers stack to this stack:
```
Executing Line | Code | Stack
---------------+---------------------------------------+--------
1| def delegatee(a, b) = a + b | self
2| | 1
3| def delegator(...) | 2
4| # | CI1 (argc: 2)
-> 5| delegatee(...) # CI2 (FORWARDING) | cref_or_me
6| end | specval
7| | type
8| def caller | self
9| delegator(1, 2) # CI1 (argc: 2) | 1
10| end | 2
```
The "FORWARDING" call site combines information from CI1 with CI2 in order
to support passing other values in addition to the `...` value, as well as
perfectly forward splat args, kwargs, etc.
Since we're able to copy the stack from `caller` in to `delegator`'s stack, we
can avoid allocating objects.
I want to do this to eliminate object allocations for delegate methods.
My long term goal is to implement `Class#new` in Ruby and it uses `...`.
I was able to implement `Class#new` in Ruby
[here](https://github.com/ruby/ruby/pull/9289).
If we adopt the technique in this patch, then we can optimize allocating
objects that take keyword parameters for `initialize`.
For example, this code will allocate 2 objects: one for `SomeObject`, and one
for the kwargs:
```ruby
SomeObject.new(foo: 1)
```
If we combine this technique, plus implement `Class#new` in Ruby, then we can
reduce allocations for this common operation.
Co-Authored-By: John Hawthorn <john@hawthorn.email>
Co-Authored-By: Alan Wu <XrXr@users.noreply.github.com>
2024-04-15 20:48:53 +03:00
|
|
|
#define VM_CALL_FORWARDING (0x01 << VM_CALL_FORWARDING_bit)
|
VALUE size packed callinfo (ci).
Now, rb_call_info contains how to call the method with tuple of
(mid, orig_argc, flags, kwarg). Most of cases, kwarg == NULL and
mid+argc+flags only requires 64bits. So this patch packed
rb_call_info to VALUE (1 word) on such cases. If we can not
represent it in VALUE, then use imemo_callinfo which contains
conventional callinfo (rb_callinfo, renamed from rb_call_info).
iseq->body->ci_kw_size is removed because all of callinfo is VALUE
size (packed ci or a pointer to imemo_callinfo).
To access ci information, we need to use these functions:
vm_ci_mid(ci), _flag(ci), _argc(ci), _kwarg(ci).
struct rb_call_info_kw_arg is renamed to rb_callinfo_kwarg.
rb_funcallv_with_cc() and rb_method_basic_definition_p_with_cc()
is temporary removed because cd->ci should be marked.
2020-01-08 02:20:36 +03:00
|
|
|
|
|
|
|
struct rb_callinfo_kwarg {
|
|
|
|
int keyword_len;
|
2023-09-30 00:06:26 +03:00
|
|
|
int references;
|
2020-04-12 22:19:06 +03:00
|
|
|
VALUE keywords[];
|
VALUE size packed callinfo (ci).
Now, rb_call_info contains how to call the method with tuple of
(mid, orig_argc, flags, kwarg). Most of cases, kwarg == NULL and
mid+argc+flags only requires 64bits. So this patch packed
rb_call_info to VALUE (1 word) on such cases. If we can not
represent it in VALUE, then use imemo_callinfo which contains
conventional callinfo (rb_callinfo, renamed from rb_call_info).
iseq->body->ci_kw_size is removed because all of callinfo is VALUE
size (packed ci or a pointer to imemo_callinfo).
To access ci information, we need to use these functions:
vm_ci_mid(ci), _flag(ci), _argc(ci), _kwarg(ci).
struct rb_call_info_kw_arg is renamed to rb_callinfo_kwarg.
rb_funcallv_with_cc() and rb_method_basic_definition_p_with_cc()
is temporary removed because cd->ci should be marked.
2020-01-08 02:20:36 +03:00
|
|
|
};
|
|
|
|
|
|
|
|
static inline size_t
|
|
|
|
rb_callinfo_kwarg_bytes(int keyword_len)
|
|
|
|
{
|
|
|
|
return rb_size_mul_add_or_raise(
|
2020-04-12 22:19:06 +03:00
|
|
|
keyword_len,
|
VALUE size packed callinfo (ci).
Now, rb_call_info contains how to call the method with tuple of
(mid, orig_argc, flags, kwarg). Most of cases, kwarg == NULL and
mid+argc+flags only requires 64bits. So this patch packed
rb_call_info to VALUE (1 word) on such cases. If we can not
represent it in VALUE, then use imemo_callinfo which contains
conventional callinfo (rb_callinfo, renamed from rb_call_info).
iseq->body->ci_kw_size is removed because all of callinfo is VALUE
size (packed ci or a pointer to imemo_callinfo).
To access ci information, we need to use these functions:
vm_ci_mid(ci), _flag(ci), _argc(ci), _kwarg(ci).
struct rb_call_info_kw_arg is renamed to rb_callinfo_kwarg.
rb_funcallv_with_cc() and rb_method_basic_definition_p_with_cc()
is temporary removed because cd->ci should be marked.
2020-01-08 02:20:36 +03:00
|
|
|
sizeof(VALUE),
|
|
|
|
sizeof(struct rb_callinfo_kwarg),
|
|
|
|
rb_eRuntimeError);
|
|
|
|
}
|
|
|
|
|
|
|
|
// imemo_callinfo
|
|
|
|
struct rb_callinfo {
|
|
|
|
VALUE flags;
|
|
|
|
const struct rb_callinfo_kwarg *kwarg;
|
|
|
|
VALUE mid;
|
|
|
|
VALUE flag;
|
|
|
|
VALUE argc;
|
|
|
|
};
|
|
|
|
|
2023-06-28 16:45:34 +03:00
|
|
|
#if !defined(USE_EMBED_CI) || (USE_EMBED_CI+0)
|
|
|
|
#undef USE_EMBED_CI
|
VALUE size packed callinfo (ci).
Now, rb_call_info contains how to call the method with tuple of
(mid, orig_argc, flags, kwarg). Most of cases, kwarg == NULL and
mid+argc+flags only requires 64bits. So this patch packed
rb_call_info to VALUE (1 word) on such cases. If we can not
represent it in VALUE, then use imemo_callinfo which contains
conventional callinfo (rb_callinfo, renamed from rb_call_info).
iseq->body->ci_kw_size is removed because all of callinfo is VALUE
size (packed ci or a pointer to imemo_callinfo).
To access ci information, we need to use these functions:
vm_ci_mid(ci), _flag(ci), _argc(ci), _kwarg(ci).
struct rb_call_info_kw_arg is renamed to rb_callinfo_kwarg.
rb_funcallv_with_cc() and rb_method_basic_definition_p_with_cc()
is temporary removed because cd->ci should be marked.
2020-01-08 02:20:36 +03:00
|
|
|
#define USE_EMBED_CI 1
|
2023-06-28 16:45:34 +03:00
|
|
|
#else
|
|
|
|
#undef USE_EMBED_CI
|
|
|
|
#define USE_EMBED_CI 0
|
VALUE size packed callinfo (ci).
Now, rb_call_info contains how to call the method with tuple of
(mid, orig_argc, flags, kwarg). Most of cases, kwarg == NULL and
mid+argc+flags only requires 64bits. So this patch packed
rb_call_info to VALUE (1 word) on such cases. If we can not
represent it in VALUE, then use imemo_callinfo which contains
conventional callinfo (rb_callinfo, renamed from rb_call_info).
iseq->body->ci_kw_size is removed because all of callinfo is VALUE
size (packed ci or a pointer to imemo_callinfo).
To access ci information, we need to use these functions:
vm_ci_mid(ci), _flag(ci), _argc(ci), _kwarg(ci).
struct rb_call_info_kw_arg is renamed to rb_callinfo_kwarg.
rb_funcallv_with_cc() and rb_method_basic_definition_p_with_cc()
is temporary removed because cd->ci should be marked.
2020-01-08 02:20:36 +03:00
|
|
|
#endif
|
|
|
|
|
|
|
|
#if SIZEOF_VALUE == 8
|
|
|
|
#define CI_EMBED_TAG_bits 1
|
|
|
|
#define CI_EMBED_ARGC_bits 15
|
|
|
|
#define CI_EMBED_FLAG_bits 16
|
|
|
|
#define CI_EMBED_ID_bits 32
|
|
|
|
#elif SIZEOF_VALUE == 4
|
|
|
|
#define CI_EMBED_TAG_bits 1
|
Reduce allocations for keyword argument hashes
Previously, passing a keyword splat to a method always allocated
a hash on the caller side, and accepting arbitrary keywords in
a method allocated a separate hash on the callee side. Passing
explicit keywords to a method that accepted a keyword splat
did not allocate a hash on the caller side, but resulted in two
hashes allocated on the callee side.
This commit makes passing a single keyword splat to a method not
allocate a hash on the caller side. Passing multiple keyword
splats or a mix of explicit keywords and a keyword splat still
generates a hash on the caller side. On the callee side,
if arbitrary keywords are not accepted, it does not allocate a
hash. If arbitrary keywords are accepted, it will allocate a
hash, but this commit uses a callinfo flag to indicate whether
the caller already allocated a hash, and if so, the callee can
use the passed hash without duplicating it. So this commit
should make it so that a maximum of a single hash is allocated
during method calls.
To set the callinfo flag appropriately, method call argument
compilation checks if only a single keyword splat is given.
If only one keyword splat is given, the VM_CALL_KW_SPLAT_MUT
callinfo flag is not set, since in that case the keyword
splat is passed directly and not mutable. If more than one
splat is used, a new hash needs to be generated on the caller
side, and in that case the callinfo flag is set, indicating
the keyword splat is mutable by the callee.
In compile_hash, used for both hash and keyword argument
compilation, if compiling keyword arguments and only a
single keyword splat is used, pass the argument directly.
On the caller side, in vm_args.c, the callinfo flag needs to
be recognized and handled. Because the keyword splat
argument may not be a hash, it needs to be converted to a
hash first if not. Then, unless the callinfo flag is set,
the hash needs to be duplicated. The temporary copy of the
callinfo flag, kw_flag, is updated if a hash was duplicated,
to prevent the need to duplicate it again. If we are
converting to a hash or duplicating a hash, we need to update
the argument array, which can including duplicating the
positional splat array if one was passed. CALLER_SETUP_ARG
and a couple other places needs to be modified to handle
similar issues for other types of calls.
This includes fairly comprehensive tests for different ways
keywords are handled internally, checking that you get equal
results but that keyword splats on the caller side result in
distinct objects for keyword rest parameters.
Included are benchmarks for keyword argument calls.
Brief results when compiled without optimization:
def kw(a: 1) a end
def kws(**kw) kw end
h = {a: 1}
kw(a: 1) # about same
kw(**h) # 2.37x faster
kws(a: 1) # 1.30x faster
kws(**h) # 2.19x faster
kw(a: 1, **h) # 1.03x slower
kw(**h, **h) # about same
kws(a: 1, **h) # 1.16x faster
kws(**h, **h) # 1.14x faster
2020-02-24 23:05:07 +03:00
|
|
|
#define CI_EMBED_ARGC_bits 3
|
|
|
|
#define CI_EMBED_FLAG_bits 13
|
VALUE size packed callinfo (ci).
Now, rb_call_info contains how to call the method with tuple of
(mid, orig_argc, flags, kwarg). Most of cases, kwarg == NULL and
mid+argc+flags only requires 64bits. So this patch packed
rb_call_info to VALUE (1 word) on such cases. If we can not
represent it in VALUE, then use imemo_callinfo which contains
conventional callinfo (rb_callinfo, renamed from rb_call_info).
iseq->body->ci_kw_size is removed because all of callinfo is VALUE
size (packed ci or a pointer to imemo_callinfo).
To access ci information, we need to use these functions:
vm_ci_mid(ci), _flag(ci), _argc(ci), _kwarg(ci).
struct rb_call_info_kw_arg is renamed to rb_callinfo_kwarg.
rb_funcallv_with_cc() and rb_method_basic_definition_p_with_cc()
is temporary removed because cd->ci should be marked.
2020-01-08 02:20:36 +03:00
|
|
|
#define CI_EMBED_ID_bits 15
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#if (CI_EMBED_TAG_bits + CI_EMBED_ARGC_bits + CI_EMBED_FLAG_bits + CI_EMBED_ID_bits) != (SIZEOF_VALUE * 8)
|
|
|
|
#error
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#define CI_EMBED_FLAG 0x01
|
|
|
|
#define CI_EMBED_ARGC_SHFT (CI_EMBED_TAG_bits)
|
2020-01-08 10:14:01 +03:00
|
|
|
#define CI_EMBED_ARGC_MASK ((((VALUE)1)<<CI_EMBED_ARGC_bits) - 1)
|
VALUE size packed callinfo (ci).
Now, rb_call_info contains how to call the method with tuple of
(mid, orig_argc, flags, kwarg). Most of cases, kwarg == NULL and
mid+argc+flags only requires 64bits. So this patch packed
rb_call_info to VALUE (1 word) on such cases. If we can not
represent it in VALUE, then use imemo_callinfo which contains
conventional callinfo (rb_callinfo, renamed from rb_call_info).
iseq->body->ci_kw_size is removed because all of callinfo is VALUE
size (packed ci or a pointer to imemo_callinfo).
To access ci information, we need to use these functions:
vm_ci_mid(ci), _flag(ci), _argc(ci), _kwarg(ci).
struct rb_call_info_kw_arg is renamed to rb_callinfo_kwarg.
rb_funcallv_with_cc() and rb_method_basic_definition_p_with_cc()
is temporary removed because cd->ci should be marked.
2020-01-08 02:20:36 +03:00
|
|
|
#define CI_EMBED_FLAG_SHFT (CI_EMBED_TAG_bits + CI_EMBED_ARGC_bits)
|
2020-01-08 10:14:01 +03:00
|
|
|
#define CI_EMBED_FLAG_MASK ((((VALUE)1)<<CI_EMBED_FLAG_bits) - 1)
|
VALUE size packed callinfo (ci).
Now, rb_call_info contains how to call the method with tuple of
(mid, orig_argc, flags, kwarg). Most of cases, kwarg == NULL and
mid+argc+flags only requires 64bits. So this patch packed
rb_call_info to VALUE (1 word) on such cases. If we can not
represent it in VALUE, then use imemo_callinfo which contains
conventional callinfo (rb_callinfo, renamed from rb_call_info).
iseq->body->ci_kw_size is removed because all of callinfo is VALUE
size (packed ci or a pointer to imemo_callinfo).
To access ci information, we need to use these functions:
vm_ci_mid(ci), _flag(ci), _argc(ci), _kwarg(ci).
struct rb_call_info_kw_arg is renamed to rb_callinfo_kwarg.
rb_funcallv_with_cc() and rb_method_basic_definition_p_with_cc()
is temporary removed because cd->ci should be marked.
2020-01-08 02:20:36 +03:00
|
|
|
#define CI_EMBED_ID_SHFT (CI_EMBED_TAG_bits + CI_EMBED_ARGC_bits + CI_EMBED_FLAG_bits)
|
2020-01-08 10:14:01 +03:00
|
|
|
#define CI_EMBED_ID_MASK ((((VALUE)1)<<CI_EMBED_ID_bits) - 1)
|
VALUE size packed callinfo (ci).
Now, rb_call_info contains how to call the method with tuple of
(mid, orig_argc, flags, kwarg). Most of cases, kwarg == NULL and
mid+argc+flags only requires 64bits. So this patch packed
rb_call_info to VALUE (1 word) on such cases. If we can not
represent it in VALUE, then use imemo_callinfo which contains
conventional callinfo (rb_callinfo, renamed from rb_call_info).
iseq->body->ci_kw_size is removed because all of callinfo is VALUE
size (packed ci or a pointer to imemo_callinfo).
To access ci information, we need to use these functions:
vm_ci_mid(ci), _flag(ci), _argc(ci), _kwarg(ci).
struct rb_call_info_kw_arg is renamed to rb_callinfo_kwarg.
rb_funcallv_with_cc() and rb_method_basic_definition_p_with_cc()
is temporary removed because cd->ci should be marked.
2020-01-08 02:20:36 +03:00
|
|
|
|
2020-01-08 10:14:01 +03:00
|
|
|
static inline bool
|
VALUE size packed callinfo (ci).
Now, rb_call_info contains how to call the method with tuple of
(mid, orig_argc, flags, kwarg). Most of cases, kwarg == NULL and
mid+argc+flags only requires 64bits. So this patch packed
rb_call_info to VALUE (1 word) on such cases. If we can not
represent it in VALUE, then use imemo_callinfo which contains
conventional callinfo (rb_callinfo, renamed from rb_call_info).
iseq->body->ci_kw_size is removed because all of callinfo is VALUE
size (packed ci or a pointer to imemo_callinfo).
To access ci information, we need to use these functions:
vm_ci_mid(ci), _flag(ci), _argc(ci), _kwarg(ci).
struct rb_call_info_kw_arg is renamed to rb_callinfo_kwarg.
rb_funcallv_with_cc() and rb_method_basic_definition_p_with_cc()
is temporary removed because cd->ci should be marked.
2020-01-08 02:20:36 +03:00
|
|
|
vm_ci_packed_p(const struct rb_callinfo *ci)
|
|
|
|
{
|
2023-06-28 16:45:34 +03:00
|
|
|
if (!USE_EMBED_CI) {
|
|
|
|
return 0;
|
|
|
|
}
|
VALUE size packed callinfo (ci).
Now, rb_call_info contains how to call the method with tuple of
(mid, orig_argc, flags, kwarg). Most of cases, kwarg == NULL and
mid+argc+flags only requires 64bits. So this patch packed
rb_call_info to VALUE (1 word) on such cases. If we can not
represent it in VALUE, then use imemo_callinfo which contains
conventional callinfo (rb_callinfo, renamed from rb_call_info).
iseq->body->ci_kw_size is removed because all of callinfo is VALUE
size (packed ci or a pointer to imemo_callinfo).
To access ci information, we need to use these functions:
vm_ci_mid(ci), _flag(ci), _argc(ci), _kwarg(ci).
struct rb_call_info_kw_arg is renamed to rb_callinfo_kwarg.
rb_funcallv_with_cc() and rb_method_basic_definition_p_with_cc()
is temporary removed because cd->ci should be marked.
2020-01-08 02:20:36 +03:00
|
|
|
if (LIKELY(((VALUE)ci) & 0x01)) {
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
else {
|
2020-01-08 10:14:01 +03:00
|
|
|
VM_ASSERT(IMEMO_TYPE_P(ci, imemo_callinfo));
|
VALUE size packed callinfo (ci).
Now, rb_call_info contains how to call the method with tuple of
(mid, orig_argc, flags, kwarg). Most of cases, kwarg == NULL and
mid+argc+flags only requires 64bits. So this patch packed
rb_call_info to VALUE (1 word) on such cases. If we can not
represent it in VALUE, then use imemo_callinfo which contains
conventional callinfo (rb_callinfo, renamed from rb_call_info).
iseq->body->ci_kw_size is removed because all of callinfo is VALUE
size (packed ci or a pointer to imemo_callinfo).
To access ci information, we need to use these functions:
vm_ci_mid(ci), _flag(ci), _argc(ci), _kwarg(ci).
struct rb_call_info_kw_arg is renamed to rb_callinfo_kwarg.
rb_funcallv_with_cc() and rb_method_basic_definition_p_with_cc()
is temporary removed because cd->ci should be marked.
2020-01-08 02:20:36 +03:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-01-08 10:14:01 +03:00
|
|
|
static inline bool
|
|
|
|
vm_ci_p(const struct rb_callinfo *ci)
|
|
|
|
{
|
|
|
|
if (vm_ci_packed_p(ci) || IMEMO_TYPE_P(ci, imemo_callinfo)) {
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
else {
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
VALUE size packed callinfo (ci).
Now, rb_call_info contains how to call the method with tuple of
(mid, orig_argc, flags, kwarg). Most of cases, kwarg == NULL and
mid+argc+flags only requires 64bits. So this patch packed
rb_call_info to VALUE (1 word) on such cases. If we can not
represent it in VALUE, then use imemo_callinfo which contains
conventional callinfo (rb_callinfo, renamed from rb_call_info).
iseq->body->ci_kw_size is removed because all of callinfo is VALUE
size (packed ci or a pointer to imemo_callinfo).
To access ci information, we need to use these functions:
vm_ci_mid(ci), _flag(ci), _argc(ci), _kwarg(ci).
struct rb_call_info_kw_arg is renamed to rb_callinfo_kwarg.
rb_funcallv_with_cc() and rb_method_basic_definition_p_with_cc()
is temporary removed because cd->ci should be marked.
2020-01-08 02:20:36 +03:00
|
|
|
static inline ID
|
|
|
|
vm_ci_mid(const struct rb_callinfo *ci)
|
|
|
|
{
|
|
|
|
if (vm_ci_packed_p(ci)) {
|
|
|
|
return (((VALUE)ci) >> CI_EMBED_ID_SHFT) & CI_EMBED_ID_MASK;
|
|
|
|
}
|
|
|
|
else {
|
|
|
|
return (ID)ci->mid;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline unsigned int
|
|
|
|
vm_ci_flag(const struct rb_callinfo *ci)
|
|
|
|
{
|
|
|
|
if (vm_ci_packed_p(ci)) {
|
|
|
|
return (unsigned int)((((VALUE)ci) >> CI_EMBED_FLAG_SHFT) & CI_EMBED_FLAG_MASK);
|
|
|
|
}
|
|
|
|
else {
|
|
|
|
return (unsigned int)ci->flag;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline unsigned int
|
|
|
|
vm_ci_argc(const struct rb_callinfo *ci)
|
|
|
|
{
|
|
|
|
if (vm_ci_packed_p(ci)) {
|
|
|
|
return (unsigned int)((((VALUE)ci) >> CI_EMBED_ARGC_SHFT) & CI_EMBED_ARGC_MASK);
|
|
|
|
}
|
|
|
|
else {
|
|
|
|
return (unsigned int)ci->argc;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline const struct rb_callinfo_kwarg *
|
|
|
|
vm_ci_kwarg(const struct rb_callinfo *ci)
|
|
|
|
{
|
|
|
|
if (vm_ci_packed_p(ci)) {
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
else {
|
|
|
|
return ci->kwarg;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void
|
|
|
|
vm_ci_dump(const struct rb_callinfo *ci)
|
|
|
|
{
|
|
|
|
if (vm_ci_packed_p(ci)) {
|
2021-09-09 17:21:06 +03:00
|
|
|
ruby_debug_printf("packed_ci ID:%s flag:%x argc:%u\n",
|
|
|
|
rb_id2name(vm_ci_mid(ci)), vm_ci_flag(ci), vm_ci_argc(ci));
|
VALUE size packed callinfo (ci).
Now, rb_call_info contains how to call the method with tuple of
(mid, orig_argc, flags, kwarg). Most of cases, kwarg == NULL and
mid+argc+flags only requires 64bits. So this patch packed
rb_call_info to VALUE (1 word) on such cases. If we can not
represent it in VALUE, then use imemo_callinfo which contains
conventional callinfo (rb_callinfo, renamed from rb_call_info).
iseq->body->ci_kw_size is removed because all of callinfo is VALUE
size (packed ci or a pointer to imemo_callinfo).
To access ci information, we need to use these functions:
vm_ci_mid(ci), _flag(ci), _argc(ci), _kwarg(ci).
struct rb_call_info_kw_arg is renamed to rb_callinfo_kwarg.
rb_funcallv_with_cc() and rb_method_basic_definition_p_with_cc()
is temporary removed because cd->ci should be marked.
2020-01-08 02:20:36 +03:00
|
|
|
}
|
|
|
|
else {
|
|
|
|
rp(ci);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
#define vm_ci_new(mid, flag, argc, kwarg) vm_ci_new_(mid, flag, argc, kwarg, __FILE__, __LINE__)
|
|
|
|
#define vm_ci_new_runtime(mid, flag, argc, kwarg) vm_ci_new_runtime_(mid, flag, argc, kwarg, __FILE__, __LINE__)
|
|
|
|
|
2022-01-27 12:43:38 +03:00
|
|
|
/* This is passed to STATIC_ASSERT. Cannot be an inline function. */
|
2020-05-29 10:04:53 +03:00
|
|
|
#define VM_CI_EMBEDDABLE_P(mid, flag, argc, kwarg) \
|
|
|
|
(((mid ) & ~CI_EMBED_ID_MASK) ? false : \
|
|
|
|
((flag) & ~CI_EMBED_FLAG_MASK) ? false : \
|
|
|
|
((argc) & ~CI_EMBED_ARGC_MASK) ? false : \
|
|
|
|
(kwarg) ? false : true)
|
|
|
|
|
2020-06-03 12:25:13 +03:00
|
|
|
#define vm_ci_new_id(mid, flag, argc, must_zero) \
|
|
|
|
((const struct rb_callinfo *) \
|
|
|
|
((((VALUE)(mid )) << CI_EMBED_ID_SHFT) | \
|
|
|
|
(((VALUE)(flag)) << CI_EMBED_FLAG_SHFT) | \
|
|
|
|
(((VALUE)(argc)) << CI_EMBED_ARGC_SHFT) | \
|
|
|
|
RUBY_FIXNUM_FLAG))
|
|
|
|
|
2024-02-12 08:43:38 +03:00
|
|
|
// vm_method.c
|
|
|
|
const struct rb_callinfo *rb_vm_ci_lookup(ID mid, unsigned int flag, unsigned int argc, const struct rb_callinfo_kwarg *kwarg);
|
|
|
|
void rb_vm_ci_free(const struct rb_callinfo *);
|
|
|
|
|
VALUE size packed callinfo (ci).
Now, rb_call_info contains how to call the method with tuple of
(mid, orig_argc, flags, kwarg). Most of cases, kwarg == NULL and
mid+argc+flags only requires 64bits. So this patch packed
rb_call_info to VALUE (1 word) on such cases. If we can not
represent it in VALUE, then use imemo_callinfo which contains
conventional callinfo (rb_callinfo, renamed from rb_call_info).
iseq->body->ci_kw_size is removed because all of callinfo is VALUE
size (packed ci or a pointer to imemo_callinfo).
To access ci information, we need to use these functions:
vm_ci_mid(ci), _flag(ci), _argc(ci), _kwarg(ci).
struct rb_call_info_kw_arg is renamed to rb_callinfo_kwarg.
rb_funcallv_with_cc() and rb_method_basic_definition_p_with_cc()
is temporary removed because cd->ci should be marked.
2020-01-08 02:20:36 +03:00
|
|
|
static inline const struct rb_callinfo *
|
|
|
|
vm_ci_new_(ID mid, unsigned int flag, unsigned int argc, const struct rb_callinfo_kwarg *kwarg, const char *file, int line)
|
|
|
|
{
|
2023-06-28 16:45:34 +03:00
|
|
|
if (USE_EMBED_CI && VM_CI_EMBEDDABLE_P(mid, flag, argc, kwarg)) {
|
VALUE size packed callinfo (ci).
Now, rb_call_info contains how to call the method with tuple of
(mid, orig_argc, flags, kwarg). Most of cases, kwarg == NULL and
mid+argc+flags only requires 64bits. So this patch packed
rb_call_info to VALUE (1 word) on such cases. If we can not
represent it in VALUE, then use imemo_callinfo which contains
conventional callinfo (rb_callinfo, renamed from rb_call_info).
iseq->body->ci_kw_size is removed because all of callinfo is VALUE
size (packed ci or a pointer to imemo_callinfo).
To access ci information, we need to use these functions:
vm_ci_mid(ci), _flag(ci), _argc(ci), _kwarg(ci).
struct rb_call_info_kw_arg is renamed to rb_callinfo_kwarg.
rb_funcallv_with_cc() and rb_method_basic_definition_p_with_cc()
is temporary removed because cd->ci should be marked.
2020-01-08 02:20:36 +03:00
|
|
|
RB_DEBUG_COUNTER_INC(ci_packed);
|
2020-06-03 12:25:13 +03:00
|
|
|
return vm_ci_new_id(mid, flag, argc, kwarg);
|
VALUE size packed callinfo (ci).
Now, rb_call_info contains how to call the method with tuple of
(mid, orig_argc, flags, kwarg). Most of cases, kwarg == NULL and
mid+argc+flags only requires 64bits. So this patch packed
rb_call_info to VALUE (1 word) on such cases. If we can not
represent it in VALUE, then use imemo_callinfo which contains
conventional callinfo (rb_callinfo, renamed from rb_call_info).
iseq->body->ci_kw_size is removed because all of callinfo is VALUE
size (packed ci or a pointer to imemo_callinfo).
To access ci information, we need to use these functions:
vm_ci_mid(ci), _flag(ci), _argc(ci), _kwarg(ci).
struct rb_call_info_kw_arg is renamed to rb_callinfo_kwarg.
rb_funcallv_with_cc() and rb_method_basic_definition_p_with_cc()
is temporary removed because cd->ci should be marked.
2020-01-08 02:20:36 +03:00
|
|
|
}
|
2020-01-08 10:14:01 +03:00
|
|
|
|
VALUE size packed callinfo (ci).
Now, rb_call_info contains how to call the method with tuple of
(mid, orig_argc, flags, kwarg). Most of cases, kwarg == NULL and
mid+argc+flags only requires 64bits. So this patch packed
rb_call_info to VALUE (1 word) on such cases. If we can not
represent it in VALUE, then use imemo_callinfo which contains
conventional callinfo (rb_callinfo, renamed from rb_call_info).
iseq->body->ci_kw_size is removed because all of callinfo is VALUE
size (packed ci or a pointer to imemo_callinfo).
To access ci information, we need to use these functions:
vm_ci_mid(ci), _flag(ci), _argc(ci), _kwarg(ci).
struct rb_call_info_kw_arg is renamed to rb_callinfo_kwarg.
rb_funcallv_with_cc() and rb_method_basic_definition_p_with_cc()
is temporary removed because cd->ci should be marked.
2020-01-08 02:20:36 +03:00
|
|
|
const bool debug = 0;
|
2021-09-09 17:21:06 +03:00
|
|
|
if (debug) ruby_debug_printf("%s:%d ", file, line);
|
2020-01-08 10:14:01 +03:00
|
|
|
|
2024-02-12 08:43:38 +03:00
|
|
|
const struct rb_callinfo *ci = rb_vm_ci_lookup(mid, flag, argc, kwarg);
|
2024-02-20 23:58:10 +03:00
|
|
|
|
VALUE size packed callinfo (ci).
Now, rb_call_info contains how to call the method with tuple of
(mid, orig_argc, flags, kwarg). Most of cases, kwarg == NULL and
mid+argc+flags only requires 64bits. So this patch packed
rb_call_info to VALUE (1 word) on such cases. If we can not
represent it in VALUE, then use imemo_callinfo which contains
conventional callinfo (rb_callinfo, renamed from rb_call_info).
iseq->body->ci_kw_size is removed because all of callinfo is VALUE
size (packed ci or a pointer to imemo_callinfo).
To access ci information, we need to use these functions:
vm_ci_mid(ci), _flag(ci), _argc(ci), _kwarg(ci).
struct rb_call_info_kw_arg is renamed to rb_callinfo_kwarg.
rb_funcallv_with_cc() and rb_method_basic_definition_p_with_cc()
is temporary removed because cd->ci should be marked.
2020-01-08 02:20:36 +03:00
|
|
|
if (debug) rp(ci);
|
|
|
|
if (kwarg) {
|
|
|
|
RB_DEBUG_COUNTER_INC(ci_kw);
|
|
|
|
}
|
|
|
|
else {
|
|
|
|
RB_DEBUG_COUNTER_INC(ci_nokw);
|
|
|
|
}
|
|
|
|
|
|
|
|
VM_ASSERT(vm_ci_flag(ci) == flag);
|
|
|
|
VM_ASSERT(vm_ci_argc(ci) == argc);
|
|
|
|
|
|
|
|
return ci;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
static inline const struct rb_callinfo *
|
|
|
|
vm_ci_new_runtime_(ID mid, unsigned int flag, unsigned int argc, const struct rb_callinfo_kwarg *kwarg, const char *file, int line)
|
|
|
|
{
|
|
|
|
RB_DEBUG_COUNTER_INC(ci_runtime);
|
|
|
|
return vm_ci_new_(mid, flag, argc, kwarg, file, line);
|
|
|
|
}
|
2020-01-08 10:14:01 +03:00
|
|
|
|
2020-05-29 09:20:57 +03:00
|
|
|
#define VM_CALLINFO_NOT_UNDER_GC IMEMO_FL_USER0
|
|
|
|
|
Optimized forwarding callers and callees
This patch optimizes forwarding callers and callees. It only optimizes methods that only take `...` as their parameter, and then pass `...` to other calls.
Calls it optimizes look like this:
```ruby
def bar(a) = a
def foo(...) = bar(...) # optimized
foo(123)
```
```ruby
def bar(a) = a
def foo(...) = bar(1, 2, ...) # optimized
foo(123)
```
```ruby
def bar(*a) = a
def foo(...)
list = [1, 2]
bar(*list, ...) # optimized
end
foo(123)
```
All variants of the above but using `super` are also optimized, including a bare super like this:
```ruby
def foo(...)
super
end
```
This patch eliminates intermediate allocations made when calling methods that accept `...`.
We can observe allocation elimination like this:
```ruby
def m
x = GC.stat(:total_allocated_objects)
yield
GC.stat(:total_allocated_objects) - x
end
def bar(a) = a
def foo(...) = bar(...)
def test
m { foo(123) }
end
test
p test # allocates 1 object on master, but 0 objects with this patch
```
```ruby
def bar(a, b:) = a + b
def foo(...) = bar(...)
def test
m { foo(1, b: 2) }
end
test
p test # allocates 2 objects on master, but 0 objects with this patch
```
How does it work?
-----------------
This patch works by using a dynamic stack size when passing forwarded parameters to callees.
The caller's info object (known as the "CI") contains the stack size of the
parameters, so we pass the CI object itself as a parameter to the callee.
When forwarding parameters, the forwarding ISeq uses the caller's CI to determine how much stack to copy, then copies the caller's stack before calling the callee.
The CI at the forwarded call site is adjusted using information from the caller's CI.
I think this description is kind of confusing, so let's walk through an example with code.
```ruby
def delegatee(a, b) = a + b
def delegator(...)
delegatee(...) # CI2 (FORWARDING)
end
def caller
delegator(1, 2) # CI1 (argc: 2)
end
```
Before we call the delegator method, the stack looks like this:
```
Executing Line | Code | Stack
---------------+---------------------------------------+--------
1| def delegatee(a, b) = a + b | self
2| | 1
3| def delegator(...) | 2
4| # |
5| delegatee(...) # CI2 (FORWARDING) |
6| end |
7| |
8| def caller |
-> 9| delegator(1, 2) # CI1 (argc: 2) |
10| end |
```
The ISeq for `delegator` is tagged as "forwardable", so when `caller` calls in
to `delegator`, it writes `CI1` on to the stack as a local variable for the
`delegator` method. The `delegator` method has a special local called `...`
that holds the caller's CI object.
Here is the ISeq disasm fo `delegator`:
```
== disasm: #<ISeq:delegator@-e:1 (1,0)-(1,39)>
local table (size: 1, argc: 0 [opts: 0, rest: -1, post: 0, block: -1, kw: -1@-1, kwrest: -1])
[ 1] "..."@0
0000 putself ( 1)[LiCa]
0001 getlocal_WC_0 "..."@0
0003 send <calldata!mid:delegatee, argc:0, FCALL|FORWARDING>, nil
0006 leave [Re]
```
The local called `...` will contain the caller's CI: CI1.
Here is the stack when we enter `delegator`:
```
Executing Line | Code | Stack
---------------+---------------------------------------+--------
1| def delegatee(a, b) = a + b | self
2| | 1
3| def delegator(...) | 2
-> 4| # | CI1 (argc: 2)
5| delegatee(...) # CI2 (FORWARDING) | cref_or_me
6| end | specval
7| | type
8| def caller |
9| delegator(1, 2) # CI1 (argc: 2) |
10| end |
```
The CI at `delegatee` on line 5 is tagged as "FORWARDING", so it knows to
memcopy the caller's stack before calling `delegatee`. In this case, it will
memcopy self, 1, and 2 to the stack before calling `delegatee`. It knows how much
memory to copy from the caller because `CI1` contains stack size information
(argc: 2).
Before executing the `send` instruction, we push `...` on the stack. The
`send` instruction pops `...`, and because it is tagged with `FORWARDING`, it
knows to memcopy (using the information in the CI it just popped):
```
== disasm: #<ISeq:delegator@-e:1 (1,0)-(1,39)>
local table (size: 1, argc: 0 [opts: 0, rest: -1, post: 0, block: -1, kw: -1@-1, kwrest: -1])
[ 1] "..."@0
0000 putself ( 1)[LiCa]
0001 getlocal_WC_0 "..."@0
0003 send <calldata!mid:delegatee, argc:0, FCALL|FORWARDING>, nil
0006 leave [Re]
```
Instruction 001 puts the caller's CI on the stack. `send` is tagged with
FORWARDING, so it reads the CI and _copies_ the callers stack to this stack:
```
Executing Line | Code | Stack
---------------+---------------------------------------+--------
1| def delegatee(a, b) = a + b | self
2| | 1
3| def delegator(...) | 2
4| # | CI1 (argc: 2)
-> 5| delegatee(...) # CI2 (FORWARDING) | cref_or_me
6| end | specval
7| | type
8| def caller | self
9| delegator(1, 2) # CI1 (argc: 2) | 1
10| end | 2
```
The "FORWARDING" call site combines information from CI1 with CI2 in order
to support passing other values in addition to the `...` value, as well as
perfectly forward splat args, kwargs, etc.
Since we're able to copy the stack from `caller` in to `delegator`'s stack, we
can avoid allocating objects.
I want to do this to eliminate object allocations for delegate methods.
My long term goal is to implement `Class#new` in Ruby and it uses `...`.
I was able to implement `Class#new` in Ruby
[here](https://github.com/ruby/ruby/pull/9289).
If we adopt the technique in this patch, then we can optimize allocating
objects that take keyword parameters for `initialize`.
For example, this code will allocate 2 objects: one for `SomeObject`, and one
for the kwargs:
```ruby
SomeObject.new(foo: 1)
```
If we combine this technique, plus implement `Class#new` in Ruby, then we can
reduce allocations for this common operation.
Co-Authored-By: John Hawthorn <john@hawthorn.email>
Co-Authored-By: Alan Wu <XrXr@users.noreply.github.com>
2024-04-15 20:48:53 +03:00
|
|
|
static inline bool
|
|
|
|
vm_ci_markable(const struct rb_callinfo *ci)
|
|
|
|
{
|
|
|
|
if (! ci) {
|
|
|
|
return false; /* or true? This is Qfalse... */
|
|
|
|
}
|
|
|
|
else if (vm_ci_packed_p(ci)) {
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
else {
|
|
|
|
VM_ASSERT(IMEMO_TYPE_P(ci, imemo_callinfo));
|
|
|
|
return ! FL_ANY_RAW((VALUE)ci, VM_CALLINFO_NOT_UNDER_GC);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-06-01 10:01:30 +03:00
|
|
|
#define VM_CI_ON_STACK(mid_, flags_, argc_, kwarg_) \
|
|
|
|
(struct rb_callinfo) { \
|
|
|
|
.flags = T_IMEMO | \
|
|
|
|
(imemo_callinfo << FL_USHIFT) | \
|
|
|
|
VM_CALLINFO_NOT_UNDER_GC, \
|
|
|
|
.mid = mid_, \
|
|
|
|
.flag = flags_, \
|
|
|
|
.argc = argc_, \
|
|
|
|
.kwarg = kwarg_, \
|
|
|
|
}
|
|
|
|
|
2020-01-08 10:14:01 +03:00
|
|
|
typedef VALUE (*vm_call_handler)(
|
|
|
|
struct rb_execution_context_struct *ec,
|
|
|
|
struct rb_control_frame_struct *cfp,
|
2020-12-14 23:40:38 +03:00
|
|
|
struct rb_calling_info *calling);
|
2020-01-08 10:14:01 +03:00
|
|
|
|
|
|
|
// imemo_callcache
|
|
|
|
|
|
|
|
struct rb_callcache {
|
2020-02-22 03:59:23 +03:00
|
|
|
const VALUE flags;
|
2020-01-08 10:14:01 +03:00
|
|
|
|
|
|
|
/* inline cache: key */
|
|
|
|
const VALUE klass; // should not mark it because klass can not be free'd
|
|
|
|
// because of this marking. When klass is collected,
|
|
|
|
// cc will be cleared (cc->klass = 0) at vm_ccs_free().
|
|
|
|
|
|
|
|
/* inline cache: values */
|
|
|
|
const struct rb_callable_method_entry_struct * const cme_;
|
|
|
|
const vm_call_handler call_;
|
|
|
|
|
|
|
|
union {
|
2022-10-03 18:14:32 +03:00
|
|
|
struct {
|
2022-10-03 20:52:40 +03:00
|
|
|
uintptr_t value; // Shape ID in upper bits, index in lower bits
|
2022-10-03 18:14:32 +03:00
|
|
|
} attr;
|
2020-01-08 10:14:01 +03:00
|
|
|
const enum method_missing_reason method_missing_reason; /* used by method_missing */
|
2020-03-02 08:25:35 +03:00
|
|
|
VALUE v;
|
2023-03-09 19:30:30 +03:00
|
|
|
const struct rb_builtin_function *bf;
|
2020-01-08 10:14:01 +03:00
|
|
|
} aux_;
|
|
|
|
};
|
|
|
|
|
2022-10-03 18:14:32 +03:00
|
|
|
#define VM_CALLCACHE_UNMARKABLE FL_FREEZE
|
|
|
|
#define VM_CALLCACHE_ON_STACK FL_EXIVAR
|
|
|
|
|
2023-10-27 03:03:17 +03:00
|
|
|
/* VM_CALLCACHE_IVAR used for IVAR/ATTRSET/STRUCT_AREF/STRUCT_ASET methods */
|
2023-07-31 10:17:55 +03:00
|
|
|
#define VM_CALLCACHE_IVAR IMEMO_FL_USER0
|
|
|
|
#define VM_CALLCACHE_BF IMEMO_FL_USER1
|
|
|
|
#define VM_CALLCACHE_SUPER IMEMO_FL_USER2
|
|
|
|
#define VM_CALLCACHE_REFINEMENT IMEMO_FL_USER3
|
2023-07-31 06:26:27 +03:00
|
|
|
|
|
|
|
enum vm_cc_type {
|
|
|
|
cc_type_normal, // chained from ccs
|
|
|
|
cc_type_super,
|
2023-07-31 10:17:55 +03:00
|
|
|
cc_type_refinement,
|
2023-07-31 06:26:27 +03:00
|
|
|
};
|
|
|
|
|
2022-10-03 18:14:32 +03:00
|
|
|
extern const struct rb_callcache *rb_vm_empty_cc(void);
|
|
|
|
extern const struct rb_callcache *rb_vm_empty_cc_for_super(void);
|
|
|
|
|
|
|
|
#define vm_cc_empty() rb_vm_empty_cc()
|
|
|
|
|
2022-10-12 13:38:29 +03:00
|
|
|
static inline void vm_cc_attr_index_set(const struct rb_callcache *cc, attr_index_t index, shape_id_t dest_shape_id);
|
|
|
|
|
2022-10-03 18:14:32 +03:00
|
|
|
static inline void
|
|
|
|
vm_cc_attr_index_initialize(const struct rb_callcache *cc, shape_id_t shape_id)
|
|
|
|
{
|
2022-10-12 13:38:29 +03:00
|
|
|
vm_cc_attr_index_set(cc, (attr_index_t)-1, shape_id);
|
2022-10-03 18:14:32 +03:00
|
|
|
}
|
2020-01-08 10:14:01 +03:00
|
|
|
|
|
|
|
static inline const struct rb_callcache *
|
|
|
|
vm_cc_new(VALUE klass,
|
|
|
|
const struct rb_callable_method_entry_struct *cme,
|
2023-07-31 06:26:27 +03:00
|
|
|
vm_call_handler call,
|
|
|
|
enum vm_cc_type type)
|
2020-01-08 10:14:01 +03:00
|
|
|
{
|
2024-02-20 23:58:10 +03:00
|
|
|
struct rb_callcache *cc = IMEMO_NEW(struct rb_callcache, imemo_callcache, klass);
|
|
|
|
*((struct rb_callable_method_entry_struct **)&cc->cme_) = (struct rb_callable_method_entry_struct *)cme;
|
|
|
|
*((vm_call_handler *)&cc->call_) = call;
|
2023-07-31 06:26:27 +03:00
|
|
|
|
Optimized forwarding callers and callees
This patch optimizes forwarding callers and callees. It only optimizes methods that only take `...` as their parameter, and then pass `...` to other calls.
Calls it optimizes look like this:
```ruby
def bar(a) = a
def foo(...) = bar(...) # optimized
foo(123)
```
```ruby
def bar(a) = a
def foo(...) = bar(1, 2, ...) # optimized
foo(123)
```
```ruby
def bar(*a) = a
def foo(...)
list = [1, 2]
bar(*list, ...) # optimized
end
foo(123)
```
All variants of the above but using `super` are also optimized, including a bare super like this:
```ruby
def foo(...)
super
end
```
This patch eliminates intermediate allocations made when calling methods that accept `...`.
We can observe allocation elimination like this:
```ruby
def m
x = GC.stat(:total_allocated_objects)
yield
GC.stat(:total_allocated_objects) - x
end
def bar(a) = a
def foo(...) = bar(...)
def test
m { foo(123) }
end
test
p test # allocates 1 object on master, but 0 objects with this patch
```
```ruby
def bar(a, b:) = a + b
def foo(...) = bar(...)
def test
m { foo(1, b: 2) }
end
test
p test # allocates 2 objects on master, but 0 objects with this patch
```
How does it work?
-----------------
This patch works by using a dynamic stack size when passing forwarded parameters to callees.
The caller's info object (known as the "CI") contains the stack size of the
parameters, so we pass the CI object itself as a parameter to the callee.
When forwarding parameters, the forwarding ISeq uses the caller's CI to determine how much stack to copy, then copies the caller's stack before calling the callee.
The CI at the forwarded call site is adjusted using information from the caller's CI.
I think this description is kind of confusing, so let's walk through an example with code.
```ruby
def delegatee(a, b) = a + b
def delegator(...)
delegatee(...) # CI2 (FORWARDING)
end
def caller
delegator(1, 2) # CI1 (argc: 2)
end
```
Before we call the delegator method, the stack looks like this:
```
Executing Line | Code | Stack
---------------+---------------------------------------+--------
1| def delegatee(a, b) = a + b | self
2| | 1
3| def delegator(...) | 2
4| # |
5| delegatee(...) # CI2 (FORWARDING) |
6| end |
7| |
8| def caller |
-> 9| delegator(1, 2) # CI1 (argc: 2) |
10| end |
```
The ISeq for `delegator` is tagged as "forwardable", so when `caller` calls in
to `delegator`, it writes `CI1` on to the stack as a local variable for the
`delegator` method. The `delegator` method has a special local called `...`
that holds the caller's CI object.
Here is the ISeq disasm fo `delegator`:
```
== disasm: #<ISeq:delegator@-e:1 (1,0)-(1,39)>
local table (size: 1, argc: 0 [opts: 0, rest: -1, post: 0, block: -1, kw: -1@-1, kwrest: -1])
[ 1] "..."@0
0000 putself ( 1)[LiCa]
0001 getlocal_WC_0 "..."@0
0003 send <calldata!mid:delegatee, argc:0, FCALL|FORWARDING>, nil
0006 leave [Re]
```
The local called `...` will contain the caller's CI: CI1.
Here is the stack when we enter `delegator`:
```
Executing Line | Code | Stack
---------------+---------------------------------------+--------
1| def delegatee(a, b) = a + b | self
2| | 1
3| def delegator(...) | 2
-> 4| # | CI1 (argc: 2)
5| delegatee(...) # CI2 (FORWARDING) | cref_or_me
6| end | specval
7| | type
8| def caller |
9| delegator(1, 2) # CI1 (argc: 2) |
10| end |
```
The CI at `delegatee` on line 5 is tagged as "FORWARDING", so it knows to
memcopy the caller's stack before calling `delegatee`. In this case, it will
memcopy self, 1, and 2 to the stack before calling `delegatee`. It knows how much
memory to copy from the caller because `CI1` contains stack size information
(argc: 2).
Before executing the `send` instruction, we push `...` on the stack. The
`send` instruction pops `...`, and because it is tagged with `FORWARDING`, it
knows to memcopy (using the information in the CI it just popped):
```
== disasm: #<ISeq:delegator@-e:1 (1,0)-(1,39)>
local table (size: 1, argc: 0 [opts: 0, rest: -1, post: 0, block: -1, kw: -1@-1, kwrest: -1])
[ 1] "..."@0
0000 putself ( 1)[LiCa]
0001 getlocal_WC_0 "..."@0
0003 send <calldata!mid:delegatee, argc:0, FCALL|FORWARDING>, nil
0006 leave [Re]
```
Instruction 001 puts the caller's CI on the stack. `send` is tagged with
FORWARDING, so it reads the CI and _copies_ the callers stack to this stack:
```
Executing Line | Code | Stack
---------------+---------------------------------------+--------
1| def delegatee(a, b) = a + b | self
2| | 1
3| def delegator(...) | 2
4| # | CI1 (argc: 2)
-> 5| delegatee(...) # CI2 (FORWARDING) | cref_or_me
6| end | specval
7| | type
8| def caller | self
9| delegator(1, 2) # CI1 (argc: 2) | 1
10| end | 2
```
The "FORWARDING" call site combines information from CI1 with CI2 in order
to support passing other values in addition to the `...` value, as well as
perfectly forward splat args, kwargs, etc.
Since we're able to copy the stack from `caller` in to `delegator`'s stack, we
can avoid allocating objects.
I want to do this to eliminate object allocations for delegate methods.
My long term goal is to implement `Class#new` in Ruby and it uses `...`.
I was able to implement `Class#new` in Ruby
[here](https://github.com/ruby/ruby/pull/9289).
If we adopt the technique in this patch, then we can optimize allocating
objects that take keyword parameters for `initialize`.
For example, this code will allocate 2 objects: one for `SomeObject`, and one
for the kwargs:
```ruby
SomeObject.new(foo: 1)
```
If we combine this technique, plus implement `Class#new` in Ruby, then we can
reduce allocations for this common operation.
Co-Authored-By: John Hawthorn <john@hawthorn.email>
Co-Authored-By: Alan Wu <XrXr@users.noreply.github.com>
2024-04-15 20:48:53 +03:00
|
|
|
VM_ASSERT(RB_TYPE_P(klass, T_CLASS) || RB_TYPE_P(klass, T_ICLASS));
|
|
|
|
|
2023-07-31 06:26:27 +03:00
|
|
|
switch (type) {
|
|
|
|
case cc_type_normal:
|
|
|
|
break;
|
|
|
|
case cc_type_super:
|
|
|
|
*(VALUE *)&cc->flags |= VM_CALLCACHE_SUPER;
|
|
|
|
break;
|
2023-07-31 10:17:55 +03:00
|
|
|
case cc_type_refinement:
|
|
|
|
*(VALUE *)&cc->flags |= VM_CALLCACHE_REFINEMENT;
|
|
|
|
break;
|
2023-07-31 06:26:27 +03:00
|
|
|
}
|
|
|
|
|
2022-10-03 18:14:32 +03:00
|
|
|
vm_cc_attr_index_initialize(cc, INVALID_SHAPE_ID);
|
2020-01-08 10:14:01 +03:00
|
|
|
RB_DEBUG_COUNTER_INC(cc_new);
|
|
|
|
return cc;
|
|
|
|
}
|
|
|
|
|
2023-07-31 06:26:27 +03:00
|
|
|
static inline bool
|
|
|
|
vm_cc_super_p(const struct rb_callcache *cc)
|
|
|
|
{
|
|
|
|
return (cc->flags & VM_CALLCACHE_SUPER) != 0;
|
|
|
|
}
|
|
|
|
|
2023-07-31 10:17:55 +03:00
|
|
|
static inline bool
|
|
|
|
vm_cc_refinement_p(const struct rb_callcache *cc)
|
|
|
|
{
|
|
|
|
return (cc->flags & VM_CALLCACHE_REFINEMENT) != 0;
|
|
|
|
}
|
|
|
|
|
2020-06-01 10:01:30 +03:00
|
|
|
#define VM_CC_ON_STACK(clazz, call, aux, cme) \
|
|
|
|
(struct rb_callcache) { \
|
|
|
|
.flags = T_IMEMO | \
|
|
|
|
(imemo_callcache << FL_USHIFT) | \
|
2021-11-16 12:14:50 +03:00
|
|
|
VM_CALLCACHE_UNMARKABLE | \
|
|
|
|
VM_CALLCACHE_ON_STACK, \
|
2020-06-01 10:01:30 +03:00
|
|
|
.klass = clazz, \
|
|
|
|
.cme_ = cme, \
|
|
|
|
.call_ = call, \
|
|
|
|
.aux_ = aux, \
|
|
|
|
}
|
2020-01-08 10:14:01 +03:00
|
|
|
|
|
|
|
static inline bool
|
|
|
|
vm_cc_class_check(const struct rb_callcache *cc, VALUE klass)
|
|
|
|
{
|
|
|
|
VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
|
|
|
|
VM_ASSERT(cc->klass == 0 ||
|
|
|
|
RB_TYPE_P(cc->klass, T_CLASS) || RB_TYPE_P(cc->klass, T_ICLASS));
|
|
|
|
return cc->klass == klass;
|
|
|
|
}
|
|
|
|
|
2021-11-16 12:02:16 +03:00
|
|
|
static inline int
|
|
|
|
vm_cc_markable(const struct rb_callcache *cc)
|
|
|
|
{
|
|
|
|
VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
|
|
|
|
return FL_TEST_RAW((VALUE)cc, VM_CALLCACHE_UNMARKABLE) == 0;
|
|
|
|
}
|
|
|
|
|
2020-01-08 10:14:01 +03:00
|
|
|
static inline const struct rb_callable_method_entry_struct *
|
|
|
|
vm_cc_cme(const struct rb_callcache *cc)
|
|
|
|
{
|
|
|
|
VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
|
2021-11-25 06:18:15 +03:00
|
|
|
VM_ASSERT(cc->call_ == NULL || // not initialized yet
|
|
|
|
!vm_cc_markable(cc) ||
|
|
|
|
cc->cme_ != NULL);
|
2021-11-16 12:02:16 +03:00
|
|
|
|
2020-01-08 10:14:01 +03:00
|
|
|
return cc->cme_;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline vm_call_handler
|
|
|
|
vm_cc_call(const struct rb_callcache *cc)
|
|
|
|
{
|
|
|
|
VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
|
2021-11-16 11:53:33 +03:00
|
|
|
VM_ASSERT(cc->call_ != NULL);
|
2020-01-08 10:14:01 +03:00
|
|
|
return cc->call_;
|
|
|
|
}
|
|
|
|
|
2022-10-03 18:14:32 +03:00
|
|
|
static inline attr_index_t
|
2020-01-08 10:14:01 +03:00
|
|
|
vm_cc_attr_index(const struct rb_callcache *cc)
|
|
|
|
{
|
|
|
|
VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
|
2022-10-03 20:52:40 +03:00
|
|
|
return (attr_index_t)((cc->aux_.attr.value & SHAPE_FLAG_MASK) - 1);
|
2022-10-03 18:14:32 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline shape_id_t
|
|
|
|
vm_cc_attr_index_dest_shape_id(const struct rb_callcache *cc)
|
|
|
|
{
|
|
|
|
VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
|
|
|
|
|
2022-10-03 20:52:40 +03:00
|
|
|
return cc->aux_.attr.value >> SHAPE_FLAG_SHIFT;
|
2022-10-03 18:14:32 +03:00
|
|
|
}
|
|
|
|
|
2022-10-03 20:52:40 +03:00
|
|
|
static inline void
|
|
|
|
vm_cc_atomic_shape_and_index(const struct rb_callcache *cc, shape_id_t * shape_id, attr_index_t * index)
|
2022-10-03 18:14:32 +03:00
|
|
|
{
|
2022-10-03 20:52:40 +03:00
|
|
|
uintptr_t cache_value = cc->aux_.attr.value; // Atomically read 64 bits
|
|
|
|
*shape_id = (shape_id_t)(cache_value >> SHAPE_FLAG_SHIFT);
|
|
|
|
*index = (attr_index_t)(cache_value & SHAPE_FLAG_MASK) - 1;
|
|
|
|
return;
|
2022-10-03 18:14:32 +03:00
|
|
|
}
|
|
|
|
|
2022-10-03 20:52:40 +03:00
|
|
|
static inline void
|
|
|
|
vm_ic_atomic_shape_and_index(const struct iseq_inline_iv_cache_entry *ic, shape_id_t * shape_id, attr_index_t * index)
|
2022-06-06 17:48:42 +03:00
|
|
|
{
|
2022-10-03 20:52:40 +03:00
|
|
|
uintptr_t cache_value = ic->value; // Atomically read 64 bits
|
|
|
|
*shape_id = (shape_id_t)(cache_value >> SHAPE_FLAG_SHIFT);
|
|
|
|
*index = (attr_index_t)(cache_value & SHAPE_FLAG_MASK) - 1;
|
|
|
|
return;
|
2022-10-03 18:14:32 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline shape_id_t
|
|
|
|
vm_ic_attr_index_dest_shape_id(const struct iseq_inline_iv_cache_entry *ic)
|
|
|
|
{
|
2022-10-03 20:52:40 +03:00
|
|
|
return (shape_id_t)(ic->value >> SHAPE_FLAG_SHIFT);
|
2022-06-06 17:48:42 +03:00
|
|
|
}
|
|
|
|
|
2020-01-08 10:14:01 +03:00
|
|
|
static inline unsigned int
|
|
|
|
vm_cc_cmethod_missing_reason(const struct rb_callcache *cc)
|
|
|
|
{
|
|
|
|
VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
|
|
|
|
return cc->aux_.method_missing_reason;
|
|
|
|
}
|
|
|
|
|
2021-01-06 07:06:25 +03:00
|
|
|
static inline bool
|
|
|
|
vm_cc_invalidated_p(const struct rb_callcache *cc)
|
|
|
|
{
|
2021-01-18 20:47:04 +03:00
|
|
|
if (cc->klass && !METHOD_ENTRY_INVALIDATED(vm_cc_cme(cc))) {
|
2021-01-06 07:06:25 +03:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
else {
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-03-07 10:15:30 +03:00
|
|
|
// For RJIT. cc_cme is supposed to have inlined `vm_cc_cme(cc)`.
|
2020-01-08 10:14:01 +03:00
|
|
|
static inline bool
|
2020-03-11 10:54:18 +03:00
|
|
|
vm_cc_valid_p(const struct rb_callcache *cc, const rb_callable_method_entry_t *cc_cme, VALUE klass)
|
2020-01-08 10:14:01 +03:00
|
|
|
{
|
|
|
|
VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
|
2020-03-11 10:54:18 +03:00
|
|
|
if (cc->klass == klass && !METHOD_ENTRY_INVALIDATED(cc_cme)) {
|
2020-01-08 10:14:01 +03:00
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
else {
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-10-23 01:20:35 +03:00
|
|
|
/* callcache: mutate */
|
2020-01-08 10:14:01 +03:00
|
|
|
|
|
|
|
static inline void
|
|
|
|
vm_cc_call_set(const struct rb_callcache *cc, vm_call_handler call)
|
|
|
|
{
|
|
|
|
VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
|
|
|
|
VM_ASSERT(cc != vm_cc_empty());
|
|
|
|
*(vm_call_handler *)&cc->call_ = call;
|
|
|
|
}
|
|
|
|
|
2023-10-27 03:03:17 +03:00
|
|
|
static inline void
|
|
|
|
set_vm_cc_ivar(const struct rb_callcache *cc)
|
|
|
|
{
|
|
|
|
*(VALUE *)&cc->flags |= VM_CALLCACHE_IVAR;
|
|
|
|
}
|
|
|
|
|
2020-01-08 10:14:01 +03:00
|
|
|
static inline void
|
2022-10-03 20:52:40 +03:00
|
|
|
vm_cc_attr_index_set(const struct rb_callcache *cc, attr_index_t index, shape_id_t dest_shape_id)
|
2020-01-08 10:14:01 +03:00
|
|
|
{
|
2022-10-12 13:38:29 +03:00
|
|
|
uintptr_t *attr_value = (uintptr_t *)&cc->aux_.attr.value;
|
|
|
|
if (!vm_cc_markable(cc)) {
|
|
|
|
*attr_value = (uintptr_t)INVALID_SHAPE_ID << SHAPE_FLAG_SHIFT;
|
|
|
|
return;
|
|
|
|
}
|
2020-01-08 10:14:01 +03:00
|
|
|
VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
|
|
|
|
VM_ASSERT(cc != vm_cc_empty());
|
2022-10-13 13:23:46 +03:00
|
|
|
*attr_value = (attr_index_t)(index + 1) | ((uintptr_t)(dest_shape_id) << SHAPE_FLAG_SHIFT);
|
2023-10-27 03:03:17 +03:00
|
|
|
set_vm_cc_ivar(cc);
|
2023-03-09 19:30:30 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline bool
|
|
|
|
vm_cc_ivar_p(const struct rb_callcache *cc)
|
|
|
|
{
|
2023-07-28 05:03:32 +03:00
|
|
|
return (cc->flags & VM_CALLCACHE_IVAR) != 0;
|
2022-01-26 00:04:17 +03:00
|
|
|
}
|
|
|
|
|
2022-06-06 17:48:42 +03:00
|
|
|
static inline void
|
2022-10-03 20:52:40 +03:00
|
|
|
vm_ic_attr_index_set(const rb_iseq_t *iseq, const struct iseq_inline_iv_cache_entry *ic, attr_index_t index, shape_id_t dest_shape_id)
|
2022-06-06 17:48:42 +03:00
|
|
|
{
|
2022-10-13 13:23:46 +03:00
|
|
|
*(uintptr_t *)&ic->value = ((uintptr_t)dest_shape_id << SHAPE_FLAG_SHIFT) | (attr_index_t)(index + 1);
|
2022-06-06 17:48:42 +03:00
|
|
|
}
|
|
|
|
|
2022-01-26 00:04:17 +03:00
|
|
|
static inline void
|
2022-10-03 18:14:32 +03:00
|
|
|
vm_ic_attr_index_initialize(const struct iseq_inline_iv_cache_entry *ic, shape_id_t shape_id)
|
2022-01-26 00:04:17 +03:00
|
|
|
{
|
2022-10-03 20:52:40 +03:00
|
|
|
*(uintptr_t *)&ic->value = (uintptr_t)shape_id << SHAPE_FLAG_SHIFT;
|
2020-01-08 10:14:01 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline void
|
|
|
|
vm_cc_method_missing_reason_set(const struct rb_callcache *cc, enum method_missing_reason reason)
|
|
|
|
{
|
|
|
|
VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
|
|
|
|
VM_ASSERT(cc != vm_cc_empty());
|
|
|
|
*(enum method_missing_reason *)&cc->aux_.method_missing_reason = reason;
|
|
|
|
}
|
|
|
|
|
2023-03-09 19:30:30 +03:00
|
|
|
static inline void
|
|
|
|
vm_cc_bf_set(const struct rb_callcache *cc, const struct rb_builtin_function *bf)
|
|
|
|
{
|
|
|
|
VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
|
|
|
|
VM_ASSERT(cc != vm_cc_empty());
|
|
|
|
*(const struct rb_builtin_function **)&cc->aux_.bf = bf;
|
2023-07-28 05:03:32 +03:00
|
|
|
*(VALUE *)&cc->flags |= VM_CALLCACHE_BF;
|
2023-03-09 19:30:30 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline bool
|
|
|
|
vm_cc_bf_p(const struct rb_callcache *cc)
|
|
|
|
{
|
2023-07-28 05:03:32 +03:00
|
|
|
return (cc->flags & VM_CALLCACHE_BF) != 0;
|
2023-03-09 19:30:30 +03:00
|
|
|
}
|
|
|
|
|
2021-06-29 01:00:35 +03:00
|
|
|
static inline void
|
|
|
|
vm_cc_invalidate(const struct rb_callcache *cc)
|
|
|
|
{
|
|
|
|
VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
|
|
|
|
VM_ASSERT(cc != vm_cc_empty());
|
|
|
|
VM_ASSERT(cc->klass != 0); // should be enable
|
|
|
|
|
|
|
|
*(VALUE *)&cc->klass = 0;
|
|
|
|
RB_DEBUG_COUNTER_INC(cc_ent_invalidate);
|
|
|
|
}
|
2020-01-08 10:14:01 +03:00
|
|
|
|
|
|
|
/* calldata */
|
|
|
|
|
|
|
|
struct rb_call_data {
|
|
|
|
const struct rb_callinfo *ci;
|
|
|
|
const struct rb_callcache *cc;
|
|
|
|
};
|
|
|
|
|
|
|
|
struct rb_class_cc_entries {
|
|
|
|
#if VM_CHECK_MODE > 0
|
|
|
|
VALUE debug_sig;
|
|
|
|
#endif
|
|
|
|
int capa;
|
|
|
|
int len;
|
|
|
|
const struct rb_callable_method_entry_struct *cme;
|
|
|
|
struct rb_class_cc_entries_entry {
|
2024-04-18 02:30:41 +03:00
|
|
|
unsigned int argc;
|
|
|
|
unsigned int flag;
|
2020-01-08 10:14:01 +03:00
|
|
|
const struct rb_callcache *cc;
|
|
|
|
} *entries;
|
|
|
|
};
|
|
|
|
|
|
|
|
#if VM_CHECK_MODE > 0
|
2021-12-21 00:03:51 +03:00
|
|
|
|
|
|
|
const rb_callable_method_entry_t *rb_vm_lookup_overloaded_cme(const rb_callable_method_entry_t *cme);
|
|
|
|
void rb_vm_dump_overloaded_cme_table(void);
|
|
|
|
|
2020-01-08 10:14:01 +03:00
|
|
|
static inline bool
|
|
|
|
vm_ccs_p(const struct rb_class_cc_entries *ccs)
|
|
|
|
{
|
|
|
|
return ccs->debug_sig == ~(VALUE)ccs;
|
|
|
|
}
|
`Primitive.mandatory_only?` for fast path
Compare with the C methods, A built-in methods written in Ruby is
slower if only mandatory parameters are given because it needs to
check the argumens and fill default values for optional and keyword
parameters (C methods can check the number of parameters with `argc`,
so there are no overhead). Passing mandatory arguments are common
(optional arguments are exceptional, in many cases) so it is important
to provide the fast path for such common cases.
`Primitive.mandatory_only?` is a special builtin function used with
`if` expression like that:
```ruby
def self.at(time, subsec = false, unit = :microsecond, in: nil)
if Primitive.mandatory_only?
Primitive.time_s_at1(time)
else
Primitive.time_s_at(time, subsec, unit, Primitive.arg!(:in))
end
end
```
and it makes two ISeq,
```
def self.at(time, subsec = false, unit = :microsecond, in: nil)
Primitive.time_s_at(time, subsec, unit, Primitive.arg!(:in))
end
def self.at(time)
Primitive.time_s_at1(time)
end
```
and (2) is pointed by (1). Note that `Primitive.mandatory_only?`
should be used only in a condition of an `if` statement and the
`if` statement should be equal to the methdo body (you can not
put any expression before and after the `if` statement).
A method entry with `mandatory_only?` (`Time.at` on the above case)
is marked as `iseq_overload`. When the method will be dispatch only
with mandatory arguments (`Time.at(0)` for example), make another
method entry with ISeq (2) as mandatory only method entry and it
will be cached in an inline method cache.
The idea is similar discussed in https://bugs.ruby-lang.org/issues/16254
but it only checks mandatory parameters or more, because many cases
only mandatory parameters are given. If we find other cases (optional
or keyword parameters are used frequently and it hurts performance),
we can extend the feature.
2021-11-12 20:12:20 +03:00
|
|
|
|
|
|
|
static inline bool
|
|
|
|
vm_cc_check_cme(const struct rb_callcache *cc, const rb_callable_method_entry_t *cme)
|
|
|
|
{
|
|
|
|
if (vm_cc_cme(cc) == cme ||
|
2021-12-21 00:03:51 +03:00
|
|
|
(cme->def->iseq_overload && vm_cc_cme(cc) == rb_vm_lookup_overloaded_cme(cme))) {
|
`Primitive.mandatory_only?` for fast path
Compare with the C methods, A built-in methods written in Ruby is
slower if only mandatory parameters are given because it needs to
check the argumens and fill default values for optional and keyword
parameters (C methods can check the number of parameters with `argc`,
so there are no overhead). Passing mandatory arguments are common
(optional arguments are exceptional, in many cases) so it is important
to provide the fast path for such common cases.
`Primitive.mandatory_only?` is a special builtin function used with
`if` expression like that:
```ruby
def self.at(time, subsec = false, unit = :microsecond, in: nil)
if Primitive.mandatory_only?
Primitive.time_s_at1(time)
else
Primitive.time_s_at(time, subsec, unit, Primitive.arg!(:in))
end
end
```
and it makes two ISeq,
```
def self.at(time, subsec = false, unit = :microsecond, in: nil)
Primitive.time_s_at(time, subsec, unit, Primitive.arg!(:in))
end
def self.at(time)
Primitive.time_s_at1(time)
end
```
and (2) is pointed by (1). Note that `Primitive.mandatory_only?`
should be used only in a condition of an `if` statement and the
`if` statement should be equal to the methdo body (you can not
put any expression before and after the `if` statement).
A method entry with `mandatory_only?` (`Time.at` on the above case)
is marked as `iseq_overload`. When the method will be dispatch only
with mandatory arguments (`Time.at(0)` for example), make another
method entry with ISeq (2) as mandatory only method entry and it
will be cached in an inline method cache.
The idea is similar discussed in https://bugs.ruby-lang.org/issues/16254
but it only checks mandatory parameters or more, because many cases
only mandatory parameters are given. If we find other cases (optional
or keyword parameters are used frequently and it hurts performance),
we can extend the feature.
2021-11-12 20:12:20 +03:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
else {
|
|
|
|
#if 1
|
2021-12-21 00:03:51 +03:00
|
|
|
// debug print
|
|
|
|
|
|
|
|
fprintf(stderr, "iseq_overload:%d\n", (int)cme->def->iseq_overload);
|
|
|
|
rp(cme);
|
|
|
|
rp(vm_cc_cme(cc));
|
|
|
|
rb_vm_lookup_overloaded_cme(cme);
|
`Primitive.mandatory_only?` for fast path
Compare with the C methods, A built-in methods written in Ruby is
slower if only mandatory parameters are given because it needs to
check the argumens and fill default values for optional and keyword
parameters (C methods can check the number of parameters with `argc`,
so there are no overhead). Passing mandatory arguments are common
(optional arguments are exceptional, in many cases) so it is important
to provide the fast path for such common cases.
`Primitive.mandatory_only?` is a special builtin function used with
`if` expression like that:
```ruby
def self.at(time, subsec = false, unit = :microsecond, in: nil)
if Primitive.mandatory_only?
Primitive.time_s_at1(time)
else
Primitive.time_s_at(time, subsec, unit, Primitive.arg!(:in))
end
end
```
and it makes two ISeq,
```
def self.at(time, subsec = false, unit = :microsecond, in: nil)
Primitive.time_s_at(time, subsec, unit, Primitive.arg!(:in))
end
def self.at(time)
Primitive.time_s_at1(time)
end
```
and (2) is pointed by (1). Note that `Primitive.mandatory_only?`
should be used only in a condition of an `if` statement and the
`if` statement should be equal to the methdo body (you can not
put any expression before and after the `if` statement).
A method entry with `mandatory_only?` (`Time.at` on the above case)
is marked as `iseq_overload`. When the method will be dispatch only
with mandatory arguments (`Time.at(0)` for example), make another
method entry with ISeq (2) as mandatory only method entry and it
will be cached in an inline method cache.
The idea is similar discussed in https://bugs.ruby-lang.org/issues/16254
but it only checks mandatory parameters or more, because many cases
only mandatory parameters are given. If we find other cases (optional
or keyword parameters are used frequently and it hurts performance),
we can extend the feature.
2021-11-12 20:12:20 +03:00
|
|
|
#endif
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-01-08 10:14:01 +03:00
|
|
|
#endif
|
|
|
|
|
|
|
|
// gc.c
|
|
|
|
void rb_vm_ccs_free(struct rb_class_cc_entries *ccs);
|
2020-04-10 08:11:40 +03:00
|
|
|
|
|
|
|
#endif /* RUBY_VM_CALLINFO_H */
|